Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions parquet/src/main/scala/magnolify/parquet/Schema.scala
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,25 @@ private object Schema {
s"found; written file schema had type $wf"
)
}
(wf.getLogicalTypeAnnotation, rf.getLogicalTypeAnnotation) match {
case (
w: LogicalTypeAnnotation.TimestampLogicalTypeAnnotation,
r: LogicalTypeAnnotation.TimestampLogicalTypeAnnotation
) if w.getUnit != r.getUnit =>
throw new InvalidRecordException(
s"Writer and reader Timestamp schemas do not match for field `${reader.getName}`: " +
s"writer is `$w` but reader is `$r`"
)
case (
w: LogicalTypeAnnotation.TimeLogicalTypeAnnotation,
r: LogicalTypeAnnotation.TimeLogicalTypeAnnotation
) if w.getUnit != r.getUnit =>
throw new InvalidRecordException(
s"Writer and reader Time schemas do not match for field `${reader.getName}`: " +
s"writer is `$w` but reader is `$r`"
)
case _ =>
}
case _ =>
throw new Exception(s"Unsupported type for $writer")
}
Expand Down
103 changes: 103 additions & 0 deletions parquet/src/test/scala/magnolify/parquet/SchemaSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -160,4 +160,107 @@ class SchemaSuite extends MagnolifySuite {
}
assert(e.getMessage.contains("not present"))
}

test("checkCompatibility: timestamp millis writer incompatible with micros reader") {
val writer = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 ts (TIMESTAMP(MILLIS,true));
|}""".stripMargin
)
val reader = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 ts (TIMESTAMP(MICROS,true));
|}""".stripMargin
)
val e = intercept[InvalidRecordException] {
Schema.checkCompatibility(writer, reader)
}
assert(e.getMessage.contains("timestamp types do not match"))
}

test("checkCompatibility: timestamp micros writer incompatible with millis reader") {
val writer = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 ts (TIMESTAMP(MICROS,true));
|}""".stripMargin
)
val reader = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 ts (TIMESTAMP(MILLIS,true));
|}""".stripMargin
)
val e = intercept[InvalidRecordException] {
Schema.checkCompatibility(writer, reader)
}
assert(e.getMessage.contains("timestamp types do not match"))
}

test("checkCompatibility: timestamp micros writer incompatible with nanos reader") {
val writer = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 ts (TIMESTAMP(MICROS,true));
|}""".stripMargin
)
val reader = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 ts (TIMESTAMP(NANOS,true));
|}""".stripMargin
)
val e = intercept[InvalidRecordException] {
Schema.checkCompatibility(writer, reader)
}
assert(e.getMessage.contains("timestamp types do not match"))
}

test("checkCompatibility: timestamp with matching logical types is compatible") {
val writer = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 ts (TIMESTAMP(MICROS,true));
|}""".stripMargin
)
val reader = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 ts (TIMESTAMP(MICROS,true));
|}""".stripMargin
)
Schema.checkCompatibility(writer, reader)
}

test("checkCompatibility: time micros writer incompatible with nanos reader") {
val writer = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 t (TIME(MICROS,false));
|}""".stripMargin
)
val reader = MessageTypeParser.parseMessageType(
"""message Record {
| required int64 t (TIME(NANOS,false));
|}""".stripMargin
)
val e = intercept[InvalidRecordException] {
Schema.checkCompatibility(writer, reader)
}
assert(e.getMessage.contains("Time types do not match"))
}

test("checkCompatibility: nested timestamp millis writer incompatible with micros reader") {
val writer = MessageTypeParser.parseMessageType(
"""message Record {
| required group inner {
| required int64 ts (TIMESTAMP(MILLIS,true));
| }
|}""".stripMargin
)
val reader = MessageTypeParser.parseMessageType(
"""message Record {
| required group inner {
| required int64 ts (TIMESTAMP(MICROS,true));
| }
|}""".stripMargin
)
val e = intercept[InvalidRecordException] {
Schema.checkCompatibility(writer, reader)
}
assert(e.getMessage.contains("timestamp types do not match"))
}
}
Loading