Add and integrate logical annotations for schema nodes

cpp/src/parquet/parquet.thrift

@@ -35,7 +35,7 @@ enum Type {
   BOOLEAN = 0;
   INT32 = 1;
   INT64 = 2;
-  INT96 = 3;
+  INT96 = 3;  // deprecated, only used by legacy implementations.
   FLOAT = 4;
   DOUBLE = 5;
   BYTE_ARRAY = 6;
@@ -259,9 +259,11 @@ struct DecimalType {
 /** Time units for logical types */
 struct MilliSeconds {}
 struct MicroSeconds {}
+struct NanoSeconds {}
 union TimeUnit {
   1: MilliSeconds MILLIS
   2: MicroSeconds MICROS
+  3: NanoSeconds NANOS
 }
 
 /**
@@ -277,7 +279,7 @@ struct TimestampType {
 /**
  * Time logical type annotation
  *
- * Allowed for physical types: INT32 (millis), INT64 (micros)
+ * Allowed for physical types: INT32 (millis), INT64 (micros, nanos)
  */
 struct TimeType {
   1: required bool isAdjustedToUTC
@@ -320,19 +322,27 @@ struct BsonType {
  * following table.
  */
 union LogicalType {
-  1:  StringType STRING       // use ConvertedType UTF8 if encoding is UTF-8
+  1:  StringType STRING       // use ConvertedType UTF8
   2:  MapType MAP             // use ConvertedType MAP
   3:  ListType LIST           // use ConvertedType LIST
   4:  EnumType ENUM           // use ConvertedType ENUM
   5:  DecimalType DECIMAL     // use ConvertedType DECIMAL
   6:  DateType DATE           // use ConvertedType DATE
-  7:  TimeType TIME           // use ConvertedType TIME_MICROS or TIME_MILLIS
-  8:  TimestampType TIMESTAMP // use ConvertedType TIMESTAMP_MICROS or TIMESTAMP_MILLIS
+
+  // use ConvertedType TIME_MICROS for TIME(isAdjustedToUTC = true, unit = MICROS)
+  // use ConvertedType TIME_MILLIS for TIME(isAdjustedToUTC = true, unit = MILLIS)
+  7:  TimeType TIME
+
+  // use ConvertedType TIMESTAMP_MICROS for TIMESTAMP(isAdjustedToUTC = true, unit = MICROS)
+  // use ConvertedType TIMESTAMP_MILLIS for TIMESTAMP(isAdjustedToUTC = true, unit = MILLIS)
+  8:  TimestampType TIMESTAMP
+
   // 9: reserved for INTERVAL
   10: IntType INTEGER         // use ConvertedType INT_* or UINT_*
   11: NullType UNKNOWN        // no compatible ConvertedType
   12: JsonType JSON           // use ConvertedType JSON
   13: BsonType BSON           // use ConvertedType BSON
+  14: UUIDType UUID
 }
 
 /**

cpp/src/parquet/printer.cc

@@ -198,7 +198,9 @@ void ParquetFilePrinter::JSONPrint(std::ostream& stream, std::list<int> selected
     stream << "     { \"Id\": \"" << i << "\", \"Name\": \"" << descr->name() << "\","
            << " \"PhysicalType\": \"" << TypeToString(descr->physical_type()) << "\","
            << " \"LogicalType\": \"" << LogicalTypeToString(descr->logical_type())
-           << "\" }";
+           << "\","
+           << " \"LogicalAnnotation\": " << (descr->logical_annotation())->ToJSON()
+           << " }";
     c++;
     if (c != static_cast<int>(selected_columns.size())) {
       stream << ",\n";

cpp/src/parquet/reader-test.cc

@@ -265,17 +265,17 @@ TEST(TestJSONWithLocalFile, JSONOutput) {
   "NumberOfRealColumns": "11",
   "NumberOfColumns": "11",
   "Columns": [
-     { "Id": "0", "Name": "id", "PhysicalType": "INT32", "LogicalType": "NONE" },
-     { "Id": "1", "Name": "bool_col", "PhysicalType": "BOOLEAN", "LogicalType": "NONE" },
-     { "Id": "2", "Name": "tinyint_col", "PhysicalType": "INT32", "LogicalType": "NONE" },
-     { "Id": "3", "Name": "smallint_col", "PhysicalType": "INT32", "LogicalType": "NONE" },
-     { "Id": "4", "Name": "int_col", "PhysicalType": "INT32", "LogicalType": "NONE" },
-     { "Id": "5", "Name": "bigint_col", "PhysicalType": "INT64", "LogicalType": "NONE" },
-     { "Id": "6", "Name": "float_col", "PhysicalType": "FLOAT", "LogicalType": "NONE" },
-     { "Id": "7", "Name": "double_col", "PhysicalType": "DOUBLE", "LogicalType": "NONE" },
-     { "Id": "8", "Name": "date_string_col", "PhysicalType": "BYTE_ARRAY", "LogicalType": "NONE" },
-     { "Id": "9", "Name": "string_col", "PhysicalType": "BYTE_ARRAY", "LogicalType": "NONE" },
-     { "Id": "10", "Name": "timestamp_col", "PhysicalType": "INT96", "LogicalType": "NONE" }
+     { "Id": "0", "Name": "id", "PhysicalType": "INT32", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "1", "Name": "bool_col", "PhysicalType": "BOOLEAN", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "2", "Name": "tinyint_col", "PhysicalType": "INT32", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "3", "Name": "smallint_col", "PhysicalType": "INT32", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "4", "Name": "int_col", "PhysicalType": "INT32", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "5", "Name": "bigint_col", "PhysicalType": "INT64", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "6", "Name": "float_col", "PhysicalType": "FLOAT", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "7", "Name": "double_col", "PhysicalType": "DOUBLE", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "8", "Name": "date_string_col", "PhysicalType": "BYTE_ARRAY", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "9", "Name": "string_col", "PhysicalType": "BYTE_ARRAY", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} },
+     { "Id": "10", "Name": "timestamp_col", "PhysicalType": "INT96", "LogicalType": "NONE", "LogicalAnnotation": {"Type": "None"} }
   ],
   "RowGroups": [
      {

cpp/src/parquet/schema.cc

@@ -95,7 +95,8 @@ const std::shared_ptr<ColumnPath> Node::path() const {
 
 bool Node::EqualsInternal(const Node* other) const {
   return type_ == other->type_ && name_ == other->name_ &&
-         repetition_ == other->repetition_ && logical_type_ == other->logical_type_;
+         repetition_ == other->repetition_ && logical_type_ == other->logical_type_ &&
+         logical_annotation_->Equals(*(other->logical_annotation()));
 }
 
 void Node::SetParent(const Node* parent) { parent_ = parent; }
@@ -200,6 +201,12 @@ PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetitio
       ss << " can not be applied to a primitive type";
       throw ParquetException(ss.str());
   }
+  // For forward compatibility, create an equivalent logical annotation
+  logical_annotation_ =
+      LogicalAnnotation::FromConvertedType(logical_type_, decimal_metadata_);
+  DCHECK(logical_annotation_ && !logical_annotation_->is_nested() &&
+         logical_annotation_->is_compatible(logical_type_, decimal_metadata_));
+
   if (type == Type::FIXED_LEN_BYTE_ARRAY) {
     if (length <= 0) {
       ss << "Invalid FIXED_LEN_BYTE_ARRAY length: " << length;
@@ -209,10 +216,51 @@ PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetitio
   }
 }
 
+PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetition,
+                             std::shared_ptr<const LogicalAnnotation> logical_annotation,
+                             Type::type physical_type, int physical_length, int id)
+    : Node(Node::PRIMITIVE, name, repetition, logical_annotation, id),
+      physical_type_(physical_type),
+      type_length_(physical_length) {
+  std::stringstream error;
+  if (logical_annotation_) {
+    // Check for annotation type <=> node type consistency
+    if (!logical_annotation_->is_nested()) {
+      // Check for annotation type <=> physical type consistency
+      if (logical_annotation_->is_applicable(physical_type, physical_length)) {
+        // For backward compatibility, assign equivalent legacy
+        // converted type (if possible)
+        logical_type_ = logical_annotation_->ToConvertedType(&decimal_metadata_);
+      } else {
+        error << logical_annotation_->ToString();
+        error << " can not be applied to primitive type ";
+        error << TypeToString(physical_type);
+        throw ParquetException(error.str());
+      }
+    } else {
+      error << "Nested annotation type ";
+      error << logical_annotation_->ToString();
+      error << " can not be applied to non-group node";
+      throw ParquetException(error.str());
+    }
+  } else {
+    logical_annotation_ = NoAnnotation::Make();
+    logical_type_ = logical_annotation_->ToConvertedType(&decimal_metadata_);
+  }
+  DCHECK(logical_annotation_ && !logical_annotation_->is_nested() &&
+         logical_annotation_->is_compatible(logical_type_, decimal_metadata_));
+
+  if (physical_type == Type::FIXED_LEN_BYTE_ARRAY) {
+    if (physical_length <= 0) {
+      error << "Invalid FIXED_LEN_BYTE_ARRAY length: " << physical_length;
+      throw ParquetException(error.str());
+    }
+  }
+}
+
 bool PrimitiveNode::EqualsInternal(const PrimitiveNode* other) const {
   bool is_equal = true;
-  if ((physical_type_ != other->physical_type_) ||
-      (logical_type_ != other->logical_type_)) {
+  if (physical_type_ != other->physical_type_) {
     return false;
   }
   if (logical_type_ == LogicalType::DECIMAL) {
@@ -241,6 +289,55 @@ void PrimitiveNode::VisitConst(Node::ConstVisitor* visitor) const {
 // ----------------------------------------------------------------------
 // Group node
 
+GroupNode::GroupNode(const std::string& name, Repetition::type repetition,
+                     const NodeVector& fields, LogicalType::type logical_type, int id)
+    : Node(Node::GROUP, name, repetition, logical_type, id), fields_(fields) {
+  // For forward compatibility, create an equivalent logical annotation
+  logical_annotation_ = LogicalAnnotation::FromConvertedType(logical_type_);
+  DCHECK(logical_annotation_ &&
+         (logical_annotation_->is_nested() || logical_annotation_->is_none()) &&
+         logical_annotation_->is_compatible(logical_type_));
+
+  field_name_to_idx_.clear();
+  auto field_idx = 0;
+  for (NodePtr& field : fields_) {
+    field->SetParent(this);
+    field_name_to_idx_.emplace(field->name(), field_idx++);
+  }
+}
+
+GroupNode::GroupNode(const std::string& name, Repetition::type repetition,
+                     const NodeVector& fields,
+                     std::shared_ptr<const LogicalAnnotation> logical_annotation, int id)
+    : Node(Node::GROUP, name, repetition, logical_annotation, id), fields_(fields) {
+  if (logical_annotation_) {
+    // Check for annotation type <=> node type consistency
+    if (logical_annotation_->is_nested()) {
+      // For backward compatibility, assign equivalent legacy converted type (if possible)
+      logical_type_ = logical_annotation_->ToConvertedType(nullptr);
+    } else {
+      std::stringstream error;
+      error << "Annotation type ";
+      error << logical_annotation_->ToString();
+      error << " can not be applied to group node";
+      throw ParquetException(error.str());
+    }
+  } else {
+    logical_annotation_ = NoAnnotation::Make();
+    logical_type_ = logical_annotation_->ToConvertedType(nullptr);
+  }
+  DCHECK(logical_annotation_ &&
+         (logical_annotation_->is_nested() || logical_annotation_->is_none()) &&
+         logical_annotation_->is_compatible(logical_type_));
+
+  field_name_to_idx_.clear();
+  auto field_idx = 0;
+  for (NodePtr& field : fields_) {
+    field->SetParent(this);
+    field_name_to_idx_.emplace(field->name(), field_idx++);
+  }
+}
+
 bool GroupNode::EqualsInternal(const GroupNode* other) const {
   if (this == other) {
     return true;
@@ -290,48 +387,55 @@ void GroupNode::VisitConst(Node::ConstVisitor* visitor) const { visitor->Visit(t
 // ----------------------------------------------------------------------
 // Node construction from Parquet metadata
 
-struct NodeParams {
-  explicit NodeParams(const std::string& name) : name(name) {}
-
-  const std::string& name;
-  Repetition::type repetition;
-  LogicalType::type logical_type;
-};
-
-static inline NodeParams GetNodeParams(const format::SchemaElement* element) {
-  NodeParams params(element->name);
-
-  params.repetition = FromThrift(element->repetition_type);
-  if (element->__isset.converted_type) {
-    params.logical_type = FromThrift(element->converted_type);
-  } else {
-    params.logical_type = LogicalType::NONE;
-  }
-  return params;
-}
-
 std::unique_ptr<Node> GroupNode::FromParquet(const void* opaque_element, int node_id,
                                              const NodeVector& fields) {
   const format::SchemaElement* element =
       static_cast<const format::SchemaElement*>(opaque_element);
-  NodeParams params = GetNodeParams(element);
-  return std::unique_ptr<Node>(new GroupNode(params.name, params.repetition, fields,
-                                             params.logical_type, node_id));
+
+  std::unique_ptr<GroupNode> group_node;
+  if (element->__isset.logicalType) {
+    // updated writer with logical type present
+    group_node = std::unique_ptr<GroupNode>(
+        new GroupNode(element->name, FromThrift(element->repetition_type), fields,
+                      LogicalAnnotation::FromThrift(element->logicalType), node_id));
+  } else {
+    group_node = std::unique_ptr<GroupNode>(new GroupNode(
+        element->name, FromThrift(element->repetition_type), fields,
+        (element->__isset.converted_type ? FromThrift(element->converted_type)
+                                         : LogicalType::NONE),
+        node_id));
+  }
+
+  return std::unique_ptr<Node>(group_node.release());
 }
 
 std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
                                                  int node_id) {
   const format::SchemaElement* element =
       static_cast<const format::SchemaElement*>(opaque_element);
-  NodeParams params = GetNodeParams(element);
 
-  std::unique_ptr<PrimitiveNode> result =
-      std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
-          params.name, params.repetition, FromThrift(element->type), params.logical_type,
-          element->type_length, element->precision, element->scale, node_id));
+  std::unique_ptr<PrimitiveNode> primitive_node;
+  if (element->__isset.logicalType) {
+    // updated writer with logical type present
+    primitive_node = std::unique_ptr<PrimitiveNode>(
+        new PrimitiveNode(element->name, FromThrift(element->repetition_type),
+                          LogicalAnnotation::FromThrift(element->logicalType),
+                          FromThrift(element->type), element->type_length, node_id));
+  } else if (element->__isset.converted_type) {
+    // legacy writer with logical type present
+    primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
+        element->name, FromThrift(element->repetition_type), FromThrift(element->type),
+        FromThrift(element->converted_type), element->type_length, element->precision,
+        element->scale, node_id));
+  } else {
+    // logical type not present
+    primitive_node = std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
+        element->name, FromThrift(element->repetition_type), NoAnnotation::Make(),
+        FromThrift(element->type), element->type_length, node_id));
+  }
 
   // Return as unique_ptr to the base type
-  return std::unique_ptr<Node>(result.release());
+  return std::unique_ptr<Node>(primitive_node.release());
 }
 
 void GroupNode::ToParquet(void* opaque_element) const {
@@ -342,16 +446,25 @@ void GroupNode::ToParquet(void* opaque_element) const {
   if (logical_type_ != LogicalType::NONE) {
     element->__set_converted_type(ToThrift(logical_type_));
   }
+  if (logical_annotation_ && logical_annotation_->is_serialized()) {
+    element->__set_logicalType(logical_annotation_->ToThrift());
+  }
+  return;
 }
 
 void PrimitiveNode::ToParquet(void* opaque_element) const {
   format::SchemaElement* element = static_cast<format::SchemaElement*>(opaque_element);
-
   element->__set_name(name_);
   element->__set_repetition_type(ToThrift(repetition_));
   if (logical_type_ != LogicalType::NONE) {
     element->__set_converted_type(ToThrift(logical_type_));
   }
+  if (logical_annotation_ && logical_annotation_->is_serialized() &&
+      // TODO(tpboudreau): remove the following conjunct to enable serialization
+      // of IntervalTypes after parquet.thrift recognizes them
+      !logical_annotation_->is_interval()) {
+    element->__set_logicalType(logical_annotation_->ToThrift());
+  }
   element->__set_type(ToThrift(physical_type_));
   if (physical_type_ == Type::FIXED_LEN_BYTE_ARRAY) {
     element->__set_type_length(type_length_);
@@ -360,6 +473,7 @@ void PrimitiveNode::ToParquet(void* opaque_element) const {
     element->__set_precision(decimal_metadata_.precision);
     element->__set_scale(decimal_metadata_.scale);
   }
+  return;
 }
 
 // ----------------------------------------------------------------------
@@ -531,7 +645,10 @@ static void PrintType(const PrimitiveNode* node, std::ostream& stream) {
 
 static void PrintLogicalType(const PrimitiveNode* node, std::ostream& stream) {
   auto lt = node->logical_type();
-  if (lt == LogicalType::DECIMAL) {
+  auto la = node->logical_annotation();
+  if (la && la->is_valid() && !la->is_none()) {
+    stream << " (" << la->ToString() << ")";
+  } else if (lt == LogicalType::DECIMAL) {
     stream << " (" << LogicalTypeToString(lt) << "(" << node->decimal_metadata().precision
            << "," << node->decimal_metadata().scale << "))";
   } else if (lt != LogicalType::NONE) {
@@ -555,7 +672,10 @@ void SchemaPrinter::Visit(const GroupNode* node) {
     PrintRepLevel(node->repetition(), stream_);
     stream_ << " group " << node->name();
     auto lt = node->logical_type();
-    if (lt != LogicalType::NONE) {
+    auto la = node->logical_annotation();
+    if (la && la->is_valid() && !la->is_none()) {
+      stream_ << " (" << la->ToString() << ")";
+    } else if (lt != LogicalType::NONE) {
       stream_ << " (" << LogicalTypeToString(lt) << ")";
     }
     stream_ << " {" << std::endl;

cpp/src/parquet/schema.h

@@ -78,12 +78,6 @@ struct ListEncoding {
   enum type { ONE_LEVEL, TWO_LEVEL, THREE_LEVEL };
 };
 
-struct DecimalMetadata {
-  bool isset;
-  int32_t scale;
-  int32_t precision;
-};
-
 class PARQUET_EXPORT ColumnPath {
  public:
   ColumnPath() : path_() {}
@@ -118,6 +112,15 @@ class PARQUET_EXPORT Node {
         id_(id),
         parent_(NULLPTR) {}
 
+  Node(Node::type type, const std::string& name, Repetition::type repetition,
+       std::shared_ptr<const LogicalAnnotation> logical_annotation, int id = -1)
+      : type_(type),
+        name_(name),
+        repetition_(repetition),
+        logical_annotation_(logical_annotation),
+        id_(id),
+        parent_(NULLPTR) {}
+
   virtual ~Node() {}
 
   bool is_primitive() const { return type_ == Node::PRIMITIVE; }
@@ -140,6 +143,10 @@ class PARQUET_EXPORT Node {
 
   LogicalType::type logical_type() const { return logical_type_; }
 
+  const std::shared_ptr<const LogicalAnnotation>& logical_annotation() const {
+    return logical_annotation_;
+  }
+
   int id() const { return id_; }
 
   const Node* parent() const { return parent_; }
@@ -172,6 +179,7 @@ class PARQUET_EXPORT Node {
   std::string name_;
   Repetition::type repetition_;
   LogicalType::type logical_type_;
+  std::shared_ptr<const LogicalAnnotation> logical_annotation_;
   int id_;
   // Nodes should not be shared, they have a single parent.
   const Node* parent_;
@@ -203,6 +211,13 @@ class PARQUET_EXPORT PrimitiveNode : public Node {
                                      precision, scale));
   }
 
+  static inline NodePtr Make(const std::string& name, Repetition::type repetition,
+                             std::shared_ptr<const LogicalAnnotation> logical_annotation,
+                             Type::type primitive_type, int primitive_length = -1) {
+    return NodePtr(new PrimitiveNode(name, repetition, logical_annotation, primitive_type,
+                                     primitive_length));
+  }
+
   bool Equals(const Node* other) const override;
 
   Type::type physical_type() const { return physical_type_; }
@@ -224,6 +239,10 @@ class PARQUET_EXPORT PrimitiveNode : public Node {
                 LogicalType::type logical_type = LogicalType::NONE, int length = -1,
                 int precision = -1, int scale = -1, int id = -1);
 
+  PrimitiveNode(const std::string& name, Repetition::type repetition,
+                std::shared_ptr<const LogicalAnnotation> logical_annotation,
+                Type::type primitive_type, int primitive_length = -1, int id = -1);
+
   Type::type physical_type_;
   int32_t type_length_;
   DecimalMetadata decimal_metadata_;
@@ -232,12 +251,6 @@ class PARQUET_EXPORT PrimitiveNode : public Node {
   // For FIXED_LEN_BYTE_ARRAY
   void SetTypeLength(int32_t length) { type_length_ = length; }
 
-  // For Decimal logical type: Precision and scale
-  void SetDecimalMetadata(int32_t scale, int32_t precision) {
-    decimal_metadata_.scale = scale;
-    decimal_metadata_.precision = precision;
-  }
-
   bool EqualsInternal(const PrimitiveNode* other) const;
 
   FRIEND_TEST(TestPrimitiveNode, Attrs);
@@ -257,6 +270,12 @@ class PARQUET_EXPORT GroupNode : public Node {
     return NodePtr(new GroupNode(name, repetition, fields, logical_type));
   }
 
+  static inline NodePtr Make(
+      const std::string& name, Repetition::type repetition, const NodeVector& fields,
+      std::shared_ptr<const LogicalAnnotation> logical_annotation) {
+    return NodePtr(new GroupNode(name, repetition, fields, logical_annotation));
+  }
+
   bool Equals(const Node* other) const override;
 
   NodePtr field(int i) const { return fields_[i]; }
@@ -276,15 +295,11 @@ class PARQUET_EXPORT GroupNode : public Node {
  private:
   GroupNode(const std::string& name, Repetition::type repetition,
             const NodeVector& fields, LogicalType::type logical_type = LogicalType::NONE,
-            int id = -1)
-      : Node(Node::GROUP, name, repetition, logical_type, id), fields_(fields) {
-    field_name_to_idx_.clear();
-    auto field_idx = 0;
-    for (NodePtr& field : fields_) {
-      field->SetParent(this);
-      field_name_to_idx_.emplace(field->name(), field_idx++);
-    }
-  }
+            int id = -1);
+
+  GroupNode(const std::string& name, Repetition::type repetition,
+            const NodeVector& fields,
+            std::shared_ptr<const LogicalAnnotation> logical_annotation, int id = -1);
 
   NodeVector fields_;
   bool EqualsInternal(const GroupNode* other) const;
@@ -341,10 +356,16 @@ class PARQUET_EXPORT ColumnDescriptor {
 
   LogicalType::type logical_type() const { return primitive_node_->logical_type(); }
 
+  const std::shared_ptr<const LogicalAnnotation>& logical_annotation() const {
+    return primitive_node_->logical_annotation();
+  }
+
   ColumnOrder column_order() const { return primitive_node_->column_order(); }
 
   SortOrder::type sort_order() const {
-    return GetSortOrder(logical_type(), physical_type());
+    auto la = logical_annotation();
+    auto pt = physical_type();
+    return la ? GetSortOrder(la, pt) : GetSortOrder(logical_type(), pt);
   }
 
   const std::string& name() const { return primitive_node_->name(); }

cpp/src/parquet/types-test.cc

@@ -37,6 +37,7 @@ TEST(TestTypeToString, PhysicalTypes) {
 TEST(TestLogicalTypeToString, LogicalTypes) {
   ASSERT_STREQ("NONE", LogicalTypeToString(LogicalType::NONE).c_str());
   ASSERT_STREQ("UTF8", LogicalTypeToString(LogicalType::UTF8).c_str());
+  ASSERT_STREQ("MAP", LogicalTypeToString(LogicalType::MAP).c_str());
   ASSERT_STREQ("MAP_KEY_VALUE", LogicalTypeToString(LogicalType::MAP_KEY_VALUE).c_str());
   ASSERT_STREQ("LIST", LogicalTypeToString(LogicalType::LIST).c_str());
   ASSERT_STREQ("ENUM", LogicalTypeToString(LogicalType::ENUM).c_str());

cpp/src/parquet/types.h

@@ -22,6 +22,7 @@
 #include <cstdint>
 #include <cstring>
 #include <iterator>
+#include <memory>
 #include <sstream>
 #include <string>
 
@@ -89,11 +90,328 @@ struct LogicalType {
   };
 };
 
+namespace format {
+
+class LogicalType;
+
+}
+
 // Mirrors parquet::FieldRepetitionType
 struct Repetition {
   enum type { REQUIRED = 0, OPTIONAL = 1, REPEATED = 2 };
 };
 
+// Reference:
+// parquet-mr/parquet-hadoop/src/main/java/org/apache/parquet/
+//                            format/converter/ParquetMetadataConverter.java
+// Sort order for page and column statistics. Types are associated with sort
+// orders (e.g., UTF8 columns should use UNSIGNED) and column stats are
+// aggregated using a sort order. As of parquet-format version 2.3.1, the
+// order used to aggregate stats is always SIGNED and is not stored in the
+// Parquet file. These stats are discarded for types that need unsigned.
+// See PARQUET-686.
+struct SortOrder {
+  enum type { SIGNED, UNSIGNED, UNKNOWN };
+};
+
+namespace schema {
+
+struct DecimalMetadata {
+  bool isset;
+  int32_t scale;
+  int32_t precision;
+};
+
+}  // namespace schema
+
+/// \brief Implementation of parquet.thrift LogicalType annotations.
+class PARQUET_EXPORT LogicalAnnotation {
+ public:
+  struct Type {
+    enum type {
+      UNKNOWN = 0,
+      STRING = 1,
+      MAP,
+      LIST,
+      ENUM,
+      DECIMAL,
+      DATE,
+      TIME,
+      TIMESTAMP,
+      INTERVAL,
+      INT,
+      NIL,  // Thrift NullType
+      JSON,
+      BSON,
+      UUID,
+      NONE
+    };
+  };
+
+  struct TimeUnit {
+    enum unit { UNKNOWN = 0, MILLIS = 1, MICROS, NANOS };
+  };
+
+  /// \brief If possible, return an annotation equivalent to the given legacy converted
+  /// type (and decimal metadata if applicable).
+  static std::shared_ptr<const LogicalAnnotation> FromConvertedType(
+      const parquet::LogicalType::type converted_type,
+      const parquet::schema::DecimalMetadata converted_decimal_metadata = {false, -1,
+                                                                           -1});
+
+  /// \brief Return the annotation represented by the Thrift intermediary object.
+  static std::shared_ptr<const LogicalAnnotation> FromThrift(
+      const parquet::format::LogicalType& thrift_logical_type);
+
+  /// \brief Return the explicitly requested annotation type.
+  static std::shared_ptr<const LogicalAnnotation> String();
+  static std::shared_ptr<const LogicalAnnotation> Map();
+  static std::shared_ptr<const LogicalAnnotation> List();
+  static std::shared_ptr<const LogicalAnnotation> Enum();
+  static std::shared_ptr<const LogicalAnnotation> Decimal(int32_t precision,
+                                                          int32_t scale = 0);
+  static std::shared_ptr<const LogicalAnnotation> Date();
+  static std::shared_ptr<const LogicalAnnotation> Time(
+      bool is_adjusted_to_utc, LogicalAnnotation::TimeUnit::unit time_unit);
+  static std::shared_ptr<const LogicalAnnotation> Timestamp(
+      bool is_adjusted_to_utc, LogicalAnnotation::TimeUnit::unit time_unit);
+  static std::shared_ptr<const LogicalAnnotation> Interval();
+  static std::shared_ptr<const LogicalAnnotation> Int(int bit_width, bool is_signed);
+  static std::shared_ptr<const LogicalAnnotation> Null();
+  static std::shared_ptr<const LogicalAnnotation> JSON();
+  static std::shared_ptr<const LogicalAnnotation> BSON();
+  static std::shared_ptr<const LogicalAnnotation> UUID();
+  static std::shared_ptr<const LogicalAnnotation> None();
+  static std::shared_ptr<const LogicalAnnotation> Unknown();
+
+  /// \brief Return true if this annotation is consistent with the given underlying
+  /// physical type.
+  bool is_applicable(parquet::Type::type primitive_type,
+                     int32_t primitive_length = -1) const;
+
+  /// \brief Return true if this annotation is equivalent to the given legacy converted
+  /// type (and decimal metadata if applicable).
+  bool is_compatible(parquet::LogicalType::type converted_type,
+                     parquet::schema::DecimalMetadata converted_decimal_metadata = {
+                         false, -1, -1}) const;
+
+  /// \brief If possible, return the legacy converted type (and decimal metadata if
+  /// applicable) equivalent to this annotation.
+  parquet::LogicalType::type ToConvertedType(
+      parquet::schema::DecimalMetadata* out_decimal_metadata) const;
+
+  /// \brief Return a printable representation of this annotation.
+  std::string ToString() const;
+
+  /// \brief Return a JSON representation of this annotation.
+  std::string ToJSON() const;
+
+  /// \brief Return a serializable Thrift object for this annotation.
+  parquet::format::LogicalType ToThrift() const;
+
+  /// \brief Return true if the given annotation is equivalent to this annotation.
+  bool Equals(const LogicalAnnotation& other) const;
+
+  /// \brief Return the enumerated type of this annotation.
+  LogicalAnnotation::Type::type type() const;
+
+  /// \brief Return the appropriate sort order for this annotation.
+  SortOrder::type sort_order() const;
+
+  // Type checks ...
+  bool is_string() const;
+  bool is_map() const;
+  bool is_list() const;
+  bool is_enum() const;
+  bool is_decimal() const;
+  bool is_date() const;
+  bool is_time() const;
+  bool is_timestamp() const;
+  bool is_interval() const;
+  bool is_int() const;
+  bool is_null() const;
+  bool is_JSON() const;
+  bool is_BSON() const;
+  bool is_UUID() const;
+  bool is_none() const;
+  /// \brief Return true if this annotation is of a known type.
+  bool is_valid() const;
+  bool is_invalid() const;
+  /// \brief Return true if this annotation is suitable for a schema GroupNode.
+  bool is_nested() const;
+  bool is_nonnested() const;
+  /// \brief Return true if this annotation is included in the Thrift output for its node.
+  bool is_serialized() const;
+
+  LogicalAnnotation(const LogicalAnnotation&) = delete;
+  LogicalAnnotation& operator=(const LogicalAnnotation&) = delete;
+  virtual ~LogicalAnnotation() noexcept;
+
+ protected:
+  LogicalAnnotation();
+
+  class Impl;
+  std::unique_ptr<const Impl> impl_;
+};
+
+/// \brief Allowed for physical type BYTE_ARRAY, must be encoded as UTF-8.
+class PARQUET_EXPORT StringAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  StringAnnotation() = default;
+};
+
+/// \brief Allowed for group nodes only.
+class PARQUET_EXPORT MapAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  MapAnnotation() = default;
+};
+
+/// \brief Allowed for group nodes only.
+class PARQUET_EXPORT ListAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  ListAnnotation() = default;
+};
+
+/// \brief Allowed for physical type BYTE_ARRAY, must be encoded as UTF-8.
+class PARQUET_EXPORT EnumAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  EnumAnnotation() = default;
+};
+
+/// \brief Allowed for physical type INT32, INT64, FIXED_LEN_BYTE_ARRAY, or BYTE_ARRAY,
+/// depending on the precision.
+class PARQUET_EXPORT DecimalAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make(int32_t precision,
+                                                       int32_t scale = 0);
+  int32_t precision() const;
+  int32_t scale() const;
+
+ private:
+  DecimalAnnotation() = default;
+};
+
+/// \brief Allowed for physical type INT32.
+class PARQUET_EXPORT DateAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  DateAnnotation() = default;
+};
+
+/// \brief Allowed for physical type INT32 (for MILLIS) or INT64 (for MICROS and NANOS).
+class PARQUET_EXPORT TimeAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make(
+      bool is_adjusted_to_utc, LogicalAnnotation::TimeUnit::unit time_unit);
+  bool is_adjusted_to_utc() const;
+  LogicalAnnotation::TimeUnit::unit time_unit() const;
+
+ private:
+  TimeAnnotation() = default;
+};
+
+/// \brief Allowed for physical type INT64.
+class PARQUET_EXPORT TimestampAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make(
+      bool is_adjusted_to_utc, LogicalAnnotation::TimeUnit::unit time_unit);
+  bool is_adjusted_to_utc() const;
+  LogicalAnnotation::TimeUnit::unit time_unit() const;
+
+ private:
+  TimestampAnnotation() = default;
+};
+
+/// \brief Allowed for physical type FIXED_LEN_BYTE_ARRAY with length 12
+class PARQUET_EXPORT IntervalAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  IntervalAnnotation() = default;
+};
+
+/// \brief Allowed for physical type INT32 (for bit widths 8, 16, and 32) and INT64
+/// (for bit width 64).
+class PARQUET_EXPORT IntAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make(int bit_width, bool is_signed);
+  int bit_width() const;
+  bool is_signed() const;
+
+ private:
+  IntAnnotation() = default;
+};
+
+/// \brief Allowed for any physical type.
+class PARQUET_EXPORT NullAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  NullAnnotation() = default;
+};
+
+/// \brief Allowed for physical type BYTE_ARRAY.
+class PARQUET_EXPORT JSONAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  JSONAnnotation() = default;
+};
+
+/// \brief Allowed for physical type BYTE_ARRAY.
+class PARQUET_EXPORT BSONAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  BSONAnnotation() = default;
+};
+
+/// \brief Allowed for physical type FIXED_LEN_BYTE_ARRAY with length 16,
+/// must encode raw UUID bytes.
+class PARQUET_EXPORT UUIDAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  UUIDAnnotation() = default;
+};
+
+/// \brief Allowed for any physical type.
+class PARQUET_EXPORT NoAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  NoAnnotation() = default;
+};
+
+/// \brief Allowed for any type.
+class PARQUET_EXPORT UnknownAnnotation : public LogicalAnnotation {
+ public:
+  static std::shared_ptr<const LogicalAnnotation> Make();
+
+ private:
+  UnknownAnnotation() = default;
+};
+
 // Data encodings. Mirrors parquet::Encoding
 struct Encoding {
   enum type {
@@ -122,19 +440,6 @@ struct PageType {
   enum type { DATA_PAGE, INDEX_PAGE, DICTIONARY_PAGE, DATA_PAGE_V2 };
 };
 
-// Reference:
-// parquet-mr/parquet-hadoop/src/main/java/org/apache/parquet/
-//                            format/converter/ParquetMetadataConverter.java
-// Sort order for page and column statistics. Types are associated with sort
-// orders (e.g., UTF8 columns should use UNSIGNED) and column stats are
-// aggregated using a sort order. As of parquet-format version 2.3.1, the
-// order used to aggregate stats is always SIGNED and is not stored in the
-// Parquet file. These stats are discarded for types that need unsigned.
-// See PARQUET-686.
-struct SortOrder {
-  enum type { SIGNED, UNSIGNED, UNKNOWN };
-};
-
 class ColumnOrder {
  public:
   enum type { UNDEFINED, TYPE_DEFINED_ORDER };
@@ -337,6 +642,9 @@ PARQUET_EXPORT SortOrder::type DefaultSortOrder(Type::type primitive);
 PARQUET_EXPORT SortOrder::type GetSortOrder(LogicalType::type converted,
                                             Type::type primitive);
 
+PARQUET_EXPORT SortOrder::type GetSortOrder(
+    const std::shared_ptr<const LogicalAnnotation>& annotation, Type::type primitive);
+
 }  // namespace parquet
 
 #endif  // PARQUET_TYPES_H