Skip to content

Commit e8f6a49

Browse files
julienledemjulienledem
committed
ARROW-674: [Java] Support additional Timestamp timezone metadata
Author: Julien Le Dem <julien@dremio.com> Closes apache#408 from julienledem/timestamp_md and squashes the following commits: e394526 [Julien Le Dem] ARROW-674: [Java] Support additional Timestamp timezone metadata
1 parent f67974b commit e8f6a49

4 files changed

Lines changed: 66 additions & 44 deletions

File tree

‎java/vector/src/main/codegen/data/ArrowTypes.tdd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@
6262
},
6363
{
6464
name: "Timestamp",
65-
fields: [{name: "unit", type: short, valueType: TimeUnit}]
65+
fields: [{name: "unit", type: short, valueType: TimeUnit}, {name: "timezone", type: String}]
6666
},
6767
{
6868
name: "Interval",

‎java/vector/src/main/java/org/apache/arrow/vector/types/Types.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,10 +109,10 @@ public class Types {
109109
private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
110110
private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
111111
private static final Field TIME_FIELD = new Field("", true, new Time(TimeUnit.MILLISECOND, 32), null);
112-
private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND), null);
113-
private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null);
114-
private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND), null);
115-
private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND), null);
112+
private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND, "UTC"), null);
113+
private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null);
114+
private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND, "UTC"), null);
115+
private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND, "UTC"), null);
116116
private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
117117
private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
118118
private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.SINGLE), null);
@@ -252,7 +252,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
252252
}
253253
},
254254
// time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
255-
TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND)) {
255+
TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, "UTC")) {
256256
@Override
257257
public Field getField() {
258258
return TIMESTAMPSEC_FIELD;
@@ -269,7 +269,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
269269
}
270270
},
271271
// time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
272-
TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) {
272+
TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, "UTC")) {
273273
@Override
274274
public Field getField() {
275275
return TIMESTAMPMILLI_FIELD;
@@ -286,7 +286,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
286286
}
287287
},
288288
// time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
289-
TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND)) {
289+
TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC")) {
290290
@Override
291291
public Field getField() {
292292
return TIMESTAMPMICRO_FIELD;
@@ -303,7 +303,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
303303
}
304304
},
305305
// time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC.
306-
TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND)) {
306+
TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, "UTC")) {
307307
@Override
308308
public Field getField() {
309309
return TIMESTAMPNANO_FIELD;

‎java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ public void nestedSchema() {
8181
new Field("child4.1", true, Utf8.INSTANCE, null)
8282
)));
8383
childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
84-
new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND), null),
84+
new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null),
8585
new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
8686
)));
8787
Schema initialSchema = new Schema(childrenBuilder.build());

‎java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java‎

Lines changed: 56 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,20 @@
2828
import org.apache.arrow.vector.types.IntervalUnit;
2929
import org.apache.arrow.vector.types.TimeUnit;
3030
import org.apache.arrow.vector.types.UnionMode;
31+
import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
32+
import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
33+
import org.apache.arrow.vector.types.pojo.ArrowType.Date;
34+
import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
35+
import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
36+
import org.apache.arrow.vector.types.pojo.ArrowType.Int;
37+
import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
38+
import org.apache.arrow.vector.types.pojo.ArrowType.List;
39+
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
40+
import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
41+
import org.apache.arrow.vector.types.pojo.ArrowType.Time;
42+
import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
43+
import org.apache.arrow.vector.types.pojo.ArrowType.Union;
44+
import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
3145
import org.junit.Test;
3246

3347
public class TestSchema {
@@ -43,46 +57,48 @@ private static Field field(String name, ArrowType type, Field... children) {
4357
@Test
4458
public void testComplex() throws IOException {
4559
Schema schema = new Schema(asList(
46-
field("a", false, new ArrowType.Int(8, true)),
47-
field("b", new ArrowType.Struct(),
48-
field("c", new ArrowType.Int(16, true)),
49-
field("d", new ArrowType.Utf8())),
50-
field("e", new ArrowType.List(), field(null, new ArrowType.Date())),
51-
field("f", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
52-
field("g", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
53-
field("h", new ArrowType.Interval(IntervalUnit.DAY_TIME))
60+
field("a", false, new Int(8, true)),
61+
field("b", new Struct(),
62+
field("c", new Int(16, true)),
63+
field("d", new Utf8())),
64+
field("e", new List(), field(null, new Date())),
65+
field("f", new FloatingPoint(FloatingPointPrecision.SINGLE)),
66+
field("g", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
67+
field("h", new Timestamp(TimeUnit.MICROSECOND, null)),
68+
field("i", new Interval(IntervalUnit.DAY_TIME))
5469
));
5570
roundTrip(schema);
5671
assertEquals(
57-
"Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>, e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND), h: Interval(DAY_TIME)>",
72+
"Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>, e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND, UTC), h: Timestamp(MICROSECOND, null), i: Interval(DAY_TIME)>",
5873
schema.toString());
5974
}
6075

6176
@Test
6277
public void testAll() throws IOException {
6378
Schema schema = new Schema(asList(
64-
field("a", false, new ArrowType.Null()),
65-
field("b", new ArrowType.Struct(), field("ba", new ArrowType.Null())),
66-
field("c", new ArrowType.List(), field("ca", new ArrowType.Null())),
67-
field("d", new ArrowType.Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new ArrowType.Null())),
68-
field("e", new ArrowType.Int(8, true)),
69-
field("f", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
70-
field("g", new ArrowType.Utf8()),
71-
field("h", new ArrowType.Binary()),
72-
field("i", new ArrowType.Bool()),
73-
field("j", new ArrowType.Decimal(5, 5)),
74-
field("k", new ArrowType.Date()),
75-
field("l", new ArrowType.Time(TimeUnit.MILLISECOND, 32)),
76-
field("m", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
77-
field("n", new ArrowType.Interval(IntervalUnit.DAY_TIME))
79+
field("a", false, new Null()),
80+
field("b", new Struct(), field("ba", new Null())),
81+
field("c", new List(), field("ca", new Null())),
82+
field("d", new Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new Null())),
83+
field("e", new Int(8, true)),
84+
field("f", new FloatingPoint(FloatingPointPrecision.SINGLE)),
85+
field("g", new Utf8()),
86+
field("h", new Binary()),
87+
field("i", new Bool()),
88+
field("j", new Decimal(5, 5)),
89+
field("k", new Date()),
90+
field("l", new Time(TimeUnit.MILLISECOND, 32)),
91+
field("m", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
92+
field("n", new Timestamp(TimeUnit.MICROSECOND, null)),
93+
field("o", new Interval(IntervalUnit.DAY_TIME))
7894
));
7995
roundTrip(schema);
8096
}
8197

8298
@Test
8399
public void testUnion() throws IOException {
84100
Schema schema = new Schema(asList(
85-
field("d", new ArrowType.Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new ArrowType.Null()))
101+
field("d", new Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new Null()))
86102
));
87103
roundTrip(schema);
88104
contains(schema, "Sparse");
@@ -91,20 +107,26 @@ public void testUnion() throws IOException {
91107
@Test
92108
public void testTS() throws IOException {
93109
Schema schema = new Schema(asList(
94-
field("a", new ArrowType.Timestamp(TimeUnit.SECOND)),
95-
field("b", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
96-
field("c", new ArrowType.Timestamp(TimeUnit.MICROSECOND)),
97-
field("d", new ArrowType.Timestamp(TimeUnit.NANOSECOND))
110+
field("a", new Timestamp(TimeUnit.SECOND, "UTC")),
111+
field("b", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
112+
field("c", new Timestamp(TimeUnit.MICROSECOND, "UTC")),
113+
field("d", new Timestamp(TimeUnit.NANOSECOND, "UTC")),
114+
field("e", new Timestamp(TimeUnit.SECOND, null)),
115+
field("f", new Timestamp(TimeUnit.MILLISECOND, null)),
116+
field("g", new Timestamp(TimeUnit.MICROSECOND, null)),
117+
field("h", new Timestamp(TimeUnit.NANOSECOND, null))
98118
));
99119
roundTrip(schema);
100-
contains(schema, "SECOND", "MILLISECOND", "MICROSECOND", "NANOSECOND");
120+
assertEquals(
121+
"Schema<a: Timestamp(SECOND, UTC), b: Timestamp(MILLISECOND, UTC), c: Timestamp(MICROSECOND, UTC), d: Timestamp(NANOSECOND, UTC), e: Timestamp(SECOND, null), f: Timestamp(MILLISECOND, null), g: Timestamp(MICROSECOND, null), h: Timestamp(NANOSECOND, null)>",
122+
schema.toString());
101123
}
102124

103125
@Test
104126
public void testInterval() throws IOException {
105127
Schema schema = new Schema(asList(
106-
field("a", new ArrowType.Interval(IntervalUnit.YEAR_MONTH)),
107-
field("b", new ArrowType.Interval(IntervalUnit.DAY_TIME))
128+
field("a", new Interval(IntervalUnit.YEAR_MONTH)),
129+
field("b", new Interval(IntervalUnit.DAY_TIME))
108130
));
109131
roundTrip(schema);
110132
contains(schema, "YEAR_MONTH", "DAY_TIME");
@@ -113,9 +135,9 @@ public void testInterval() throws IOException {
113135
@Test
114136
public void testFP() throws IOException {
115137
Schema schema = new Schema(asList(
116-
field("a", new ArrowType.FloatingPoint(FloatingPointPrecision.HALF)),
117-
field("b", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
118-
field("c", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))
138+
field("a", new FloatingPoint(FloatingPointPrecision.HALF)),
139+
field("b", new FloatingPoint(FloatingPointPrecision.SINGLE)),
140+
field("c", new FloatingPoint(FloatingPointPrecision.DOUBLE))
119141
));
120142
roundTrip(schema);
121143
contains(schema, "HALF", "SINGLE", "DOUBLE");

0 commit comments

Comments
 (0)