Address other format incompatibilities, write vectorLayout to Arrow metadata

Change-Id: Ic04ef893957c98ba2747f6ad9cef0e7ebe596958

cpp/src/arrow/io/io-file-test.cc

@@ -19,7 +19,7 @@
 #include <cstdio>
 #include <cstring>
 #ifndef _MSC_VER
-# include <fcntl.h>
+#include <fcntl.h>
 #endif
 #include <fstream>
 #include <memory>

cpp/src/arrow/ipc/adapter.cc

@@ -402,9 +402,6 @@ class RecordBatchReader {
 
   Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
     BufferMetadata metadata = metadata_->buffer(buffer_index);
-    if (!BitUtil::IsMultipleOf8(metadata.length)) {
-      return Status::Invalid("Expected buffer to be a multiple of 8 bytes");
-    }
     return file_->ReadAt(metadata.offset, metadata.length, out);
   }
 

cpp/src/arrow/ipc/json-internal.cc

@@ -45,8 +45,6 @@ namespace ipc {
 using RjArray = rj::Value::ConstArray;
 using RjObject = rj::Value::ConstObject;
 
-enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY };
-
 static std::string GetBufferTypeName(BufferType type) {
   switch (type) {
     case BufferType::DATA:
@@ -93,27 +91,6 @@ static std::string GetTimeUnitName(TimeUnit unit) {
   return "UNKNOWN";
 }
 
-class BufferLayout {
- public:
-  BufferLayout(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {}
-
-  BufferType type() const { return type_; }
-  int bit_width() const { return bit_width_; }
-
- private:
-  BufferType type_;
-  int bit_width_;
-};
-
-static const BufferLayout kValidityBuffer(BufferType::VALIDITY, 1);
-static const BufferLayout kOffsetBuffer(BufferType::OFFSET, 32);
-static const BufferLayout kTypeBuffer(BufferType::TYPE, 32);
-static const BufferLayout kBooleanBuffer(BufferType::DATA, 1);
-static const BufferLayout kValues64(BufferType::DATA, 64);
-static const BufferLayout kValues32(BufferType::DATA, 32);
-static const BufferLayout kValues16(BufferType::DATA, 16);
-static const BufferLayout kValues8(BufferType::DATA, 8);
-
 class JsonSchemaWriter : public TypeVisitor {
  public:
   explicit JsonSchemaWriter(const Schema& schema, RjWriter* writer)
@@ -154,9 +131,9 @@ class JsonSchemaWriter : public TypeVisitor {
   }
 
   template <typename T>
-  typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
-                              std::is_base_of<BooleanType, T>::value ||
-                              std::is_base_of<NullType, T>::value,
+  typename std::enable_if<
+      std::is_base_of<NoExtraMeta, T>::value || std::is_base_of<BooleanType, T>::value ||
+          std::is_base_of<DateType, T>::value || std::is_base_of<NullType, T>::value,
       void>::type
   WriteTypeMetadata(const T& type) {}
 
@@ -243,11 +220,10 @@ class JsonSchemaWriter : public TypeVisitor {
   }
 
   template <typename T>
-  Status WritePrimitive(const std::string& typeclass, const T& type,
-      const std::vector<BufferLayout>& buffer_layout) {
+  Status WritePrimitive(const std::string& typeclass, const T& type) {
     WriteName(typeclass, type);
     SetNoChildren();
-    WriteBufferLayout(buffer_layout);
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
@@ -255,17 +231,17 @@ class JsonSchemaWriter : public TypeVisitor {
   Status WriteVarBytes(const std::string& typeclass, const T& type) {
     WriteName(typeclass, type);
     SetNoChildren();
-    WriteBufferLayout({kValidityBuffer, kOffsetBuffer, kValues8});
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
-  void WriteBufferLayout(const std::vector<BufferLayout>& buffer_layout) {
+  void WriteBufferLayout(const std::vector<BufferDescr>& buffer_layout) {
     writer_->Key("typeLayout");
     writer_->StartObject();
     writer_->Key("vectors");
     writer_->StartArray();
 
-    for (const BufferLayout& buffer : buffer_layout) {
+    for (const BufferDescr& buffer : buffer_layout) {
       writer_->StartObject();
       writer_->Key("type");
       writer_->String(GetBufferTypeName(buffer.type()));
@@ -289,74 +265,52 @@ class JsonSchemaWriter : public TypeVisitor {
     return Status::OK();
   }
 
-  Status Visit(const NullType& type) override { return WritePrimitive("null", type, {}); }
+  Status Visit(const NullType& type) override { return WritePrimitive("null", type); }
 
-  Status Visit(const BooleanType& type) override {
-    return WritePrimitive("bool", type, {kValidityBuffer, kBooleanBuffer});
-  }
+  Status Visit(const BooleanType& type) override { return WritePrimitive("bool", type); }
 
-  Status Visit(const Int8Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues8});
-  }
+  Status Visit(const Int8Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const Int16Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues16});
-  }
+  Status Visit(const Int16Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const Int32Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues32});
-  }
+  Status Visit(const Int32Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const Int64Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues64});
-  }
+  Status Visit(const Int64Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const UInt8Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues8});
-  }
+  Status Visit(const UInt8Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const UInt16Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues16});
-  }
+  Status Visit(const UInt16Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const UInt32Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues32});
-  }
+  Status Visit(const UInt32Type& type) override { return WritePrimitive("int", type); }
 
-  Status Visit(const UInt64Type& type) override {
-    return WritePrimitive("int", type, {kValidityBuffer, kValues64});
-  }
+  Status Visit(const UInt64Type& type) override { return WritePrimitive("int", type); }
 
   Status Visit(const HalfFloatType& type) override {
-    return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues16});
+    return WritePrimitive("floatingpoint", type);
   }
 
   Status Visit(const FloatType& type) override {
-    return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues32});
+    return WritePrimitive("floatingpoint", type);
   }
 
   Status Visit(const DoubleType& type) override {
-    return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues64});
+    return WritePrimitive("floatingpoint", type);
   }
 
   Status Visit(const StringType& type) override { return WriteVarBytes("utf8", type); }
 
   Status Visit(const BinaryType& type) override { return WriteVarBytes("binary", type); }
 
-  Status Visit(const DateType& type) override {
-    return WritePrimitive("date", type, {kValidityBuffer, kValues64});
-  }
+  Status Visit(const DateType& type) override { return WritePrimitive("date", type); }
 
-  Status Visit(const TimeType& type) override {
-    return WritePrimitive("time", type, {kValidityBuffer, kValues64});
-  }
+  Status Visit(const TimeType& type) override { return WritePrimitive("time", type); }
 
   Status Visit(const TimestampType& type) override {
-    return WritePrimitive("timestamp", type, {kValidityBuffer, kValues64});
+    return WritePrimitive("timestamp", type);
   }
 
   Status Visit(const IntervalType& type) override {
-    return WritePrimitive("interval", type, {kValidityBuffer, kValues64});
+    return WritePrimitive("interval", type);
   }
 
   Status Visit(const DecimalType& type) override { return Status::NotImplemented("NYI"); }
@@ -364,26 +318,21 @@ class JsonSchemaWriter : public TypeVisitor {
   Status Visit(const ListType& type) override {
     WriteName("list", type);
     RETURN_NOT_OK(WriteChildren(type.children()));
-    WriteBufferLayout({kValidityBuffer, kOffsetBuffer});
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
   Status Visit(const StructType& type) override {
     WriteName("struct", type);
     WriteChildren(type.children());
-    WriteBufferLayout({kValidityBuffer, kTypeBuffer});
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 
   Status Visit(const UnionType& type) override {
     WriteName("union", type);
     WriteChildren(type.children());
-
-    if (type.mode == UnionMode::SPARSE) {
-      WriteBufferLayout({kValidityBuffer, kTypeBuffer});
-    } else {
-      WriteBufferLayout({kValidityBuffer, kTypeBuffer, kOffsetBuffer});
-    }
+    WriteBufferLayout(type.GetBufferLayout());
     return Status::OK();
   }
 

cpp/src/arrow/ipc/metadata-internal.cc

@@ -152,7 +152,32 @@ static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type
   break;
 
 static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
-    std::vector<FieldOffset>* children, flatbuf::Type* out_type, Offset* offset) {
+    std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
+    flatbuf::Type* out_type, Offset* offset) {
+  std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
+  for (const BufferDescr& descr : buffer_layout) {
+    flatbuf::VectorType vector_type;
+    switch (descr.type()) {
+      case BufferType::OFFSET:
+        vector_type = flatbuf::VectorType_OFFSET;
+        break;
+      case BufferType::DATA:
+        vector_type = flatbuf::VectorType_DATA;
+        break;
+      case BufferType::VALIDITY:
+        vector_type = flatbuf::VectorType_VALIDITY;
+        break;
+      case BufferType::TYPE:
+        vector_type = flatbuf::VectorType_TYPE;
+        break;
+      default:
+        vector_type = flatbuf::VectorType_DATA;
+        break;
+    }
+    auto offset = flatbuf::CreateVectorLayout(fbb, descr.bit_width(), vector_type);
+    layout->push_back(offset);
+  }
+
   switch (type->type) {
     case Type::BOOL:
       *out_type = flatbuf::Type_Bool;
@@ -211,14 +236,18 @@ static Status FieldToFlatbuffer(
 
   flatbuf::Type type_enum;
   Offset type_data;
+  Offset type_layout;
   std::vector<FieldOffset> children;
+  std::vector<VectorLayoutOffset> layout;
 
-  RETURN_NOT_OK(TypeToFlatbuffer(fbb, field->type, &children, &type_enum, &type_data));
+  RETURN_NOT_OK(
+      TypeToFlatbuffer(fbb, field->type, &children, &layout, &type_enum, &type_data));
   auto fb_children = fbb.CreateVector(children);
+  auto fb_layout = fbb.CreateVector(layout);
 
   // TODO: produce the list of VectorTypes
   *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_data,
-      field->dictionary, fb_children);
+      field->dictionary, fb_children, fb_layout);
 
   return Status::OK();
 }
@@ -293,8 +322,8 @@ Status WriteRecordBatchMetadata(int32_t length, int64_t body_length,
     const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
   flatbuffers::FlatBufferBuilder fbb;
 
-  auto batch = flatbuf::CreateRecordBatch(fbb, length, fbb.CreateVectorOfStructs(nodes),
-      fbb.CreateVectorOfStructs(buffers));
+  auto batch = flatbuf::CreateRecordBatch(
+      fbb, length, fbb.CreateVectorOfStructs(nodes), fbb.CreateVectorOfStructs(buffers));
 
   fbb.Finish(batch);
 

cpp/src/arrow/ipc/metadata-internal.h

@@ -41,6 +41,7 @@ namespace ipc {
 
 using FBB = flatbuffers::FlatBufferBuilder;
 using FieldOffset = flatbuffers::Offset<arrow::flatbuf::Field>;
+using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
 using Offset = flatbuffers::Offset<void>;
 
 static constexpr flatbuf::MetadataVersion kMetadataVersion =

cpp/src/arrow/ipc/metadata.cc

@@ -184,12 +184,13 @@ RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Message>& message
   impl_.reset(new RecordBatchMetadataImpl(message->impl_->header()));
 }
 
-RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Buffer>& buffer, int64_t offset) {
+RecordBatchMetadata::RecordBatchMetadata(
+    const std::shared_ptr<Buffer>& buffer, int64_t offset) {
   message_ = nullptr;
   buffer_ = buffer;
 
-  const flatbuf::RecordBatch* metadata = flatbuffers::GetRoot<flatbuf::RecordBatch>(
-      buffer->data() + offset);
+  const flatbuf::RecordBatch* metadata =
+      flatbuffers::GetRoot<flatbuf::RecordBatch>(buffer->data() + offset);
 
   // TODO(wesm): validate table
 

cpp/src/arrow/test-util.h

@@ -61,10 +61,10 @@
 
 // Alias MSVC popcount to GCC name
 #ifdef _MSC_VER
-#  include <intrin.h>
-#  define __builtin_popcount __popcnt
-#  include <nmmintrin.h>
-#  define __builtin_popcountll _mm_popcnt_u64
+#include <intrin.h>
+#define __builtin_popcount __popcnt
+#include <nmmintrin.h>
+#define __builtin_popcountll _mm_popcnt_u64
 #endif
 
 namespace arrow {

cpp/src/arrow/type.cc

@@ -105,10 +105,6 @@ std::string UnionType::ToString() const {
   return s.str();
 }
 
-int NullType::bit_width() const {
-  return 0;
-}
-
 std::string NullType::ToString() const {
   return name();
 }
@@ -187,4 +183,46 @@ std::shared_ptr<Field> field(
   return std::make_shared<Field>(name, type, nullable, dictionary);
 }
 
+static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1);
+static const BufferDescr kOffsetBuffer(BufferType::OFFSET, 32);
+static const BufferDescr kTypeBuffer(BufferType::TYPE, 32);
+static const BufferDescr kBooleanBuffer(BufferType::DATA, 1);
+static const BufferDescr kValues64(BufferType::DATA, 64);
+static const BufferDescr kValues32(BufferType::DATA, 32);
+static const BufferDescr kValues16(BufferType::DATA, 16);
+static const BufferDescr kValues8(BufferType::DATA, 8);
+
+std::vector<BufferDescr> FixedWidthType::GetBufferLayout() const {
+  return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
+}
+
+std::vector<BufferDescr> NullType::GetBufferLayout() const {
+  return {};
+}
+
+std::vector<BufferDescr> BinaryType::GetBufferLayout() const {
+  return {kValidityBuffer, kOffsetBuffer, kValues8};
+}
+
+std::vector<BufferDescr> ListType::GetBufferLayout() const {
+  return {kValidityBuffer, kOffsetBuffer};
+}
+
+std::vector<BufferDescr> StructType::GetBufferLayout() const {
+  return {kValidityBuffer, kTypeBuffer};
+}
+
+std::vector<BufferDescr> UnionType::GetBufferLayout() const {
+  if (mode == UnionMode::SPARSE) {
+    return {kValidityBuffer, kTypeBuffer};
+  } else {
+    return {kValidityBuffer, kTypeBuffer, kOffsetBuffer};
+  }
+}
+
+std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
+  // TODO(wesm)
+  return {};
+}
+
 }  // namespace arrow

cpp/src/arrow/type.h

@@ -101,6 +101,20 @@ struct Type {
   };
 };
 
+enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY };
+
+class BufferDescr {
+ public:
+  BufferDescr(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {}
+
+  BufferType type() const { return type_; }
+  int bit_width() const { return bit_width_; }
+
+ private:
+  BufferType type_;
+  int bit_width_;
+};
+
 struct ARROW_EXPORT DataType {
   Type::type type;
 
@@ -129,12 +143,18 @@ struct ARROW_EXPORT DataType {
   virtual Status Accept(TypeVisitor* visitor) const = 0;
 
   virtual std::string ToString() const = 0;
+
+  virtual std::vector<BufferDescr> GetBufferLayout() const = 0;
 };
 
 typedef std::shared_ptr<DataType> TypePtr;
 
-struct ARROW_EXPORT FixedWidthMeta {
+struct ARROW_EXPORT FixedWidthType : public DataType {
+  using DataType::DataType;
+
   virtual int bit_width() const = 0;
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 struct ARROW_EXPORT IntegerMeta {
@@ -184,12 +204,12 @@ struct ARROW_EXPORT Field {
 };
 typedef std::shared_ptr<Field> FieldPtr;
 
-struct ARROW_EXPORT PrimitiveCType : public DataType {
-  using DataType::DataType;
+struct ARROW_EXPORT PrimitiveCType : public FixedWidthType {
+  using FixedWidthType::FixedWidthType;
 };
 
 template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
-struct ARROW_EXPORT CTypeImpl : public PrimitiveCType, public FixedWidthMeta {
+struct ARROW_EXPORT CTypeImpl : public PrimitiveCType {
   using c_type = C_TYPE;
   static constexpr Type::type type_id = TYPE_ID;
 
@@ -204,16 +224,17 @@ struct ARROW_EXPORT CTypeImpl : public PrimitiveCType, public FixedWidthMeta {
   std::string ToString() const override { return std::string(DERIVED::name()); }
 };
 
-struct ARROW_EXPORT NullType : public DataType, public FixedWidthMeta {
+struct ARROW_EXPORT NullType : public DataType {
   static constexpr Type::type type_id = Type::NA;
 
   NullType() : DataType(Type::NA) {}
 
-  int bit_width() const override;
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
 
   static std::string name() { return "null"; }
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
@@ -221,10 +242,10 @@ struct IntegerTypeImpl : public CTypeImpl<DERIVED, TYPE_ID, C_TYPE>, public Inte
   bool is_signed() const override { return std::is_signed<C_TYPE>::value; }
 };
 
-struct ARROW_EXPORT BooleanType : public DataType, FixedWidthMeta {
+struct ARROW_EXPORT BooleanType : public FixedWidthType {
   static constexpr Type::type type_id = Type::BOOL;
 
-  BooleanType() : DataType(Type::BOOL) {}
+  BooleanType() : FixedWidthType(Type::BOOL) {}
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
@@ -306,6 +327,8 @@ struct ARROW_EXPORT ListType : public DataType, public NoExtraMeta {
   std::string ToString() const override;
 
   static std::string name() { return "list"; }
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 // BinaryType type is reprsents lists of 1-byte values.
@@ -318,6 +341,8 @@ struct ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
   std::string ToString() const override;
   static std::string name() { return "binary"; }
 
+  std::vector<BufferDescr> GetBufferLayout() const override;
+
  protected:
   // Allow subclasses to change the logical type.
   explicit BinaryType(Type::type logical_type) : DataType(logical_type) {}
@@ -345,6 +370,8 @@ struct ARROW_EXPORT StructType : public DataType, public NoExtraMeta {
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "struct"; }
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 struct ARROW_EXPORT DecimalType : public DataType {
@@ -358,6 +385,8 @@ struct ARROW_EXPORT DecimalType : public DataType {
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override;
   static std::string name() { return "decimal"; }
+
+  std::vector<BufferDescr> GetBufferLayout() const override;
 };
 
 enum class UnionMode : char { SPARSE, DENSE };
@@ -375,14 +404,20 @@ struct ARROW_EXPORT UnionType : public DataType {
   static std::string name() { return "union"; }
   Status Accept(TypeVisitor* visitor) const override;
 
+  std::vector<BufferDescr> GetBufferLayout() const override;
+
   UnionMode mode;
   std::vector<uint8_t> type_ids;
 };
 
-struct ARROW_EXPORT DateType : public DataType, public NoExtraMeta {
+struct ARROW_EXPORT DateType : public FixedWidthType {
   static constexpr Type::type type_id = Type::DATE;
 
-  DateType() : DataType(Type::DATE) {}
+  using c_type = int32_t;
+
+  DateType() : FixedWidthType(Type::DATE) {}
+
+  int bit_width() const override { return sizeof(c_type) * 8; }
 
   Status Accept(TypeVisitor* visitor) const override;
   std::string ToString() const override { return name(); }
@@ -391,13 +426,17 @@ struct ARROW_EXPORT DateType : public DataType, public NoExtraMeta {
 
 enum class TimeUnit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
 
-struct ARROW_EXPORT TimeType : public DataType {
+struct ARROW_EXPORT TimeType : public FixedWidthType {
   static constexpr Type::type type_id = Type::TIME;
   using Unit = TimeUnit;
+  using c_type = int64_t;
 
   TimeUnit unit;
 
-  explicit TimeType(TimeUnit unit = TimeUnit::MILLI) : DataType(Type::TIME), unit(unit) {}
+  int bit_width() const override { return sizeof(c_type) * 8; }
+
+  explicit TimeType(TimeUnit unit = TimeUnit::MILLI)
+      : FixedWidthType(Type::TIME), unit(unit) {}
   TimeType(const TimeType& other) : TimeType(other.unit) {}
 
   Status Accept(TypeVisitor* visitor) const override;
@@ -405,7 +444,7 @@ struct ARROW_EXPORT TimeType : public DataType {
   static std::string name() { return "time"; }
 };
 
-struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
+struct ARROW_EXPORT TimestampType : public FixedWidthType {
   using Unit = TimeUnit;
 
   typedef int64_t c_type;
@@ -416,7 +455,7 @@ struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
   TimeUnit unit;
 
   explicit TimestampType(TimeUnit unit = TimeUnit::MILLI)
-      : DataType(Type::TIMESTAMP), unit(unit) {}
+      : FixedWidthType(Type::TIMESTAMP), unit(unit) {}
 
   TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
 
@@ -425,10 +464,10 @@ struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
   static std::string name() { return "timestamp"; }
 };
 
-struct ARROW_EXPORT IntervalType : public DataType, public FixedWidthMeta {
+struct ARROW_EXPORT IntervalType : public FixedWidthType {
   enum class Unit : char { YEAR_MONTH = 0, DAY_TIME = 1 };
 
-  typedef int64_t c_type;
+  using c_type = int64_t;
   static constexpr Type::type type_id = Type::INTERVAL;
 
   int bit_width() const override { return sizeof(int64_t) * 8; }
@@ -436,7 +475,7 @@ struct ARROW_EXPORT IntervalType : public DataType, public FixedWidthMeta {
   Unit unit;
 
   explicit IntervalType(Unit unit = Unit::YEAR_MONTH)
-      : DataType(Type::INTERVAL), unit(unit) {}
+      : FixedWidthType(Type::INTERVAL), unit(unit) {}
 
   IntervalType(const IntervalType& other) : IntervalType(other.unit) {}
 

cpp/src/arrow/types/primitive.cc

@@ -49,7 +49,7 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
     const uint8_t* this_data = raw_data_;
     const uint8_t* other_data = other.raw_data_;
 
-    auto size_meta = dynamic_cast<const FixedWidthMeta*>(type_.get());
+    auto size_meta = dynamic_cast<const FixedWidthType*>(type_.get());
     int value_byte_size = size_meta->bit_width() / 8;
     DCHECK_GT(value_byte_size, 0);
 

integration/integration_test.py

@@ -80,6 +80,9 @@ class IntegrationRunner(object):
 
 class Tester(object):
 
+    def __init__(self, debug=False):
+        self.debug = debug
+
     def json_to_arrow(self, json_path, arrow_path):
         raise NotImplementedError
 
@@ -106,6 +109,10 @@ class JavaTester(Tester):
             cmd.extend(['-j', json_path])
 
         cmd.extend(['-c', command])
+
+        if self.debug:
+            print(' '.join(cmd))
+
         return run_cmd(cmd)
 
     def validate(self, json_path, arrow_path):
@@ -124,15 +131,9 @@ class CPPTester(Tester):
 
     name = 'C++'
 
-    def __init__(self, debug=False):
-        self.debug = debug
-
     def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
         cmd = [self.CPP_INTEGRATION_EXE, '--integration']
 
-        if self.debug:
-            cmd = ['gdb', '--args'] + cmd
-
         if arrow_path is not None:
             cmd.append('--arrow=' + arrow_path)
 
@@ -140,6 +141,10 @@ class CPPTester(Tester):
             cmd.append('--json=' + json_path)
 
         cmd.append('--mode=' + command)
+
+        if self.debug:
+            print(' '.join(cmd))
+
         return run_cmd(cmd)
 
     def validate(self, json_path, arrow_path):
@@ -155,7 +160,7 @@ def get_json_files():
 
 
 def run_all_tests(debug=False):
-    testers = [JavaTester(), CPPTester(debug=debug)]
+    testers = [JavaTester(debug=debug), CPPTester(debug=debug)]
     json_files = get_json_files()
 
     runner = IntegrationRunner(json_files, testers, debug=debug)

java/pom.xml

@@ -24,7 +24,7 @@
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
-  <description>Apache arrow is an open source, low latency SQL query engine for Hadoop and NoSQL.</description>
+  <description>Apache Arrow is open source, in-memory columnar data structures and low-overhead messaging</description>
   <url>http://arrow.apache.org/</url>
 
   <properties>
@@ -442,8 +442,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <!-- Mockito needs to be on the class path after JUnit (or Hamcrest) as 
-           long as Mockito _contains_ older Hamcrest classes.  See arrow-2130. --> 
+      <!-- Mockito needs to be on the class path after JUnit (or Hamcrest) as
+           long as Mockito _contains_ older Hamcrest classes.  See arrow-2130. -->
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
       <version>1.9.5</version>

java/tools/pom.xml

@@ -45,6 +45,12 @@
             <artifactId>commons-cli</artifactId>
             <version>1.2</version>
         </dependency>
+        <dependency>
+          <groupId>ch.qos.logback</groupId>
+          <artifactId>logback-classic</artifactId>
+          <version>1.0.13</version>
+          <scope>run</scope>
+        </dependency>
     </dependencies>
 
     <build>

java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java

@@ -81,7 +81,9 @@ public class VectorLoader {
     try {
       vector.loadFieldBuffers(fieldNode, ownBuffers);
     } catch (RuntimeException e) {
-      throw new IllegalArgumentException("Could not load buffers for field " + field, e);
+      e.printStackTrace();
+      throw new IllegalArgumentException("Could not load buffers for field " +
+              field + " error message" + e.getMessage(), e);
     }
     List<Field> children = field.getChildren();
     if (children.size() > 0) {

java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java

@@ -118,6 +118,7 @@ public class ArrowWriter implements AutoCloseable {
       if (startPosition != currentPosition) {
         writeZeros((int)(startPosition - currentPosition));
       }
+
       write(buffer);
       if (currentPosition != startPosition + layout.getSize()) {
         throw new IllegalStateException("wrong buffer size: " + currentPosition + " != " + startPosition + layout.getSize());
@@ -134,7 +135,9 @@ public class ArrowWriter implements AutoCloseable {
   }
 
   private void write(ArrowBuf buffer) throws IOException {
-    write(buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes()));
+    ByteBuffer nioBuffer = buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes());
+    LOGGER.debug("Writing buffer with size: " + nioBuffer.remaining());
+    write(nioBuffer);
   }
 
   private void checkStarted() throws IOException {