mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-29 09:05:47 +08:00
Code Activity

Merge remote-tracking branch 'upstream/master' into multi-batch-toPandas-ARROW-369

Browse Source
This commit is contained in:
Bryan Cutler committed 2016-12-01 09:33:16 -08:00
2 parents da65345538 3b946b8224
commit 068bc1bf90
69 files changed
+2526 -660
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+26
View File
@@ -0,0 +1,26 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Compiled source
*.a
*.dll
*.o
*.py[ocd]
*.so
*.dylib
.build_cache_dir
MANIFEST
LICENSE.txt
+12
View File
@@ -200,3 +200,15 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--------------------------------------------------------------------------------
This product includes code from Apache Kudu.
* cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake
Copyright: 2016 The Apache Software Foundation.
Home page: https://kudu.apache.org/
License: http://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
NOTICE.txt
+14
View File
@@ -41,3 +41,17 @@ This product includes software from the CMake project
This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
* Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
--------------------------------------------------------------------------------
This product includes code from Apache Kudu, which includes the following in
its NOTICE file:
Apache Kudu
Copyright 2016 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
Portions of this software were developed at
Cloudera, Inc (http://www.cloudera.com/).
appveyor.yml
+39
View File
@@ -0,0 +1,39 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Operating system (build VM template)
os: Visual Studio 2015
environment:
matrix:
- GENERATOR: Visual Studio 14 2015 Win64
# - GENERATOR: Visual Studio 14 2015
MSVC_DEFAULT_OPTIONS: ON
BOOST_ROOT: C:\Libraries\boost_1_59_0
BOOST_LIBRARYDIR: C:\Libraries\boost_1_59_0\lib64-msvc-14.0
build_script:
- cd cpp
- mkdir build
- cd build
# A lot of features are still deactivated as they do not build on Windows
# * gbenchmark doesn't build with MSVC
- cmake -G "%GENERATOR%" -DARROW_BOOST_USE_SHARED=OFF -DARROW_IPC=OFF -DARROW_HDFS=OFF -DARROW_BUILD_BENCHMARKS=OFF ..
- cmake --build . --config Debug
# test_script:
# - ctest -VV
cpp/CMakeLists.txt
+80 -28
View File
@@ -26,6 +26,7 @@ include(ExternalProject)
set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
set(GFLAGS_VERSION "2.1.2")
set(GTEST_VERSION "1.7.0")
set(GBENCHMARK_VERSION "1.0.0")
set(FLATBUFFERS_VERSION "1.3.0")
@@ -140,9 +141,11 @@ endif()
# For CMAKE_BUILD_TYPE=Release
# -O3: Enable all compiler optimizations
# -g: Enable symbols for profiler tools (TODO: remove for shipping)
set(CXX_FLAGS_DEBUG "-ggdb -O0")
set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
if (NOT MSVC)
set(CXX_FLAGS_DEBUG "-ggdb -O0")
set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG")
endif()
set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate")
set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use")
@@ -346,6 +349,8 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
COMPILE_FLAGS " -DARROW_VALGRIND")
add_test(${TEST_NAME}
valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
elseif(MSVC)
add_test(${TEST_NAME} ${TEST_PATH})
else()
add_test(${TEST_NAME}
${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
@@ -430,40 +435,44 @@ endfunction()
# ----------------------------------------------------------------------
# Add Boost dependencies (code adapted from Apache Kudu (incubating))
# find boost headers and libs
# Find static boost headers and libs
# TODO Differentiate here between release and debug builds
set(Boost_DEBUG TRUE)
set(Boost_USE_MULTITHREADED ON)
set(Boost_USE_STATIC_LIBS ON)
find_package(Boost COMPONENTS system filesystem REQUIRED)
include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
set(BOOST_STATIC_LIBS ${Boost_LIBRARIES})
list(LENGTH BOOST_STATIC_LIBS BOOST_STATIC_LIBS_LEN)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
else()
set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
endif()
# Find Boost shared libraries.
# Find shared Boost libraries.
set(Boost_USE_STATIC_LIBS OFF)
find_package(Boost COMPONENTS system filesystem REQUIRED)
set(BOOST_SHARED_LIBS ${Boost_LIBRARIES})
list(LENGTH BOOST_SHARED_LIBS BOOST_SHARED_LIBS_LEN)
list(SORT BOOST_SHARED_LIBS)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
else()
set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
endif()
message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
math(EXPR LAST_IDX "${BOOST_STATIC_LIBS_LEN} - 1")
foreach(IDX RANGE ${LAST_IDX})
list(GET BOOST_STATIC_LIBS ${IDX} BOOST_STATIC_LIB)
list(GET BOOST_SHARED_LIBS ${IDX} BOOST_SHARED_LIB)
ADD_THIRDPARTY_LIB(boost_system
STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}"
SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
ADD_THIRDPARTY_LIB(boost_filesystem
STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
SET(ARROW_BOOST_LIBS boost_system boost_filesystem)
# Remove the prefix/suffix from the library name.
#
# e.g. libboost_system-mt --> boost_system
get_filename_component(LIB_NAME ${BOOST_STATIC_LIB} NAME_WE)
string(REGEX REPLACE "lib([^-]*)(-mt)?" "\\1" LIB_NAME_NO_PREFIX_SUFFIX ${LIB_NAME})
ADD_THIRDPARTY_LIB(${LIB_NAME_NO_PREFIX_SUFFIX}
STATIC_LIB "${BOOST_STATIC_LIB}"
SHARED_LIB "${BOOST_SHARED_LIB}")
list(APPEND ARROW_BOOST_LIBS ${LIB_NAME_NO_PREFIX_SUFFIX})
endforeach()
include_directories(SYSTEM ${Boost_INCLUDE_DIR})
# ----------------------------------------------------------------------
@@ -481,7 +490,7 @@ if(ARROW_BUILD_TESTS)
ExternalProject_Add(googletest_ep
URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}
CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON
# googletest doesn't define install rules, so just build in the
# source dir and don't try to install. See its README for
# details.
@@ -490,7 +499,7 @@ if(ARROW_BUILD_TESTS)
set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
set(GTEST_STATIC_LIB "${GTEST_PREFIX}/libgtest.a")
set(GTEST_STATIC_LIB "${GTEST_PREFIX}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GTEST_VENDORED 1)
else()
find_package(GTest REQUIRED)
@@ -506,6 +515,48 @@ if(ARROW_BUILD_TESTS)
if(GTEST_VENDORED)
add_dependencies(gtest googletest_ep)
endif()
# gflags (formerly Googleflags) command line parsing
if("$ENV{GFLAGS_HOME}" STREQUAL "")
if(APPLE)
set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++")
else()
set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC")
endif()
set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
ExternalProject_Add(gflags_ep
GIT_REPOSITORY https://github.com/gflags/gflags.git
GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
BUILD_IN_SOURCE 1
CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
-DBUILD_SHARED_LIBS=OFF
-DBUILD_STATIC_LIBS=ON
-DBUILD_PACKAGING=OFF
-DBUILD_TESTING=OFF
-BUILD_CONFIG_TESTS=OFF
-DINSTALL_HEADERS=ON
-DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
set(GFLAGS_HOME "${GFLAGS_PREFIX}")
set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a")
set(GFLAGS_VENDORED 1)
else()
set(GFLAGS_VENDORED 0)
find_package(GFlags REQUIRED)
endif()
message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}")
message(STATUS "GFlags static library: ${GFLAGS_STATIC_LIB}")
include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
ADD_THIRDPARTY_LIB(gflags
STATIC_LIB ${GFLAGS_STATIC_LIB})
if(GFLAGS_VENDORED)
add_dependencies(gflags gflags_ep)
endif()
endif()
if(ARROW_BUILD_BENCHMARKS)
@@ -527,7 +578,7 @@ if(ARROW_BUILD_BENCHMARKS)
"-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}")
set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include")
set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/libbenchmark.a")
set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GBENCHMARK_VENDORED 1)
else()
find_package(GBenchmark REQUIRED)
@@ -694,6 +745,7 @@ set(ARROW_SRCS
src/arrow/array.cc
src/arrow/builder.cc
src/arrow/column.cc
src/arrow/pretty_print.cc
src/arrow/schema.cc
src/arrow/table.cc
src/arrow/type.cc
cpp/cmake_modules/CompilerInfo.cmake
+27 -15
View File
@@ -1,25 +1,32 @@
# Copyright 2013 Cloudera, Inc.
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Sets COMPILER_FAMILY to 'clang' or 'gcc'
# Sets COMPILER_VERSION to the version
execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v
ERROR_VARIABLE COMPILER_VERSION_FULL)
message(INFO " ${COMPILER_VERSION_FULL}")
message(INFO " ${CMAKE_CXX_COMPILER_ID}")
if(MSVC)
set(COMPILER_FAMILY "msvc")
# clang on Linux and Mac OS X before 10.9
if("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
set(COMPILER_FAMILY "clang")
string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1"
COMPILER_VERSION "${COMPILER_VERSION_FULL}")
@@ -29,10 +36,15 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1"
COMPILER_VERSION "${COMPILER_VERSION_FULL}")
# clang on Mac OS X, XCode 7+. No version replacement is done
# because Apple no longer advertises the upstream LLVM version.
elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-.*")
# clang on Mac OS X, XCode 7.
elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-7")
set(COMPILER_FAMILY "clang")
set(COMPILER_VERSION "3.7.0svn")
# clang on Mac OS X, XCode 8.
elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-8")
set(COMPILER_FAMILY "clang")
set(COMPILER_VERSION "3.8.0svn")
# gcc
elseif("${COMPILER_VERSION_FULL}" MATCHES ".*gcc version.*")
cpp/cmake_modules/FindGFlags.cmake
+60
View File
@@ -0,0 +1,60 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# - Find GFLAGS (gflags.h, libgflags.a, libgflags.so, and libgflags.so.0)
# This module defines
# GFLAGS_INCLUDE_DIR, directory containing headers
# GFLAGS_SHARED_LIB, path to libgflags shared library
# GFLAGS_STATIC_LIB, path to libgflags static library
# GFLAGS_FOUND, whether gflags has been found
if( NOT "$ENV{GFLAGS_HOME}" STREQUAL "")
file( TO_CMAKE_PATH "$ENV{GFLAGS_HOME}" _native_path )
list( APPEND _gflags_roots ${_native_path} )
elseif ( GFlags_HOME )
list( APPEND _gflags_roots ${GFlags_HOME} )
endif()
if ( _gflags_roots )
find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h
PATHS ${_gflags_roots}
NO_DEFAULT_PATH
PATH_SUFFIXES "include" )
find_library(GFLAGS_SHARED_LIB NAMES gflags
PATHS ${_gflags_roots}
NO_DEFAULT_PATH
PATH_SUFFIXES "lib" )
find_library(GFLAGS_SHARED_LIB NAMES libgflags.a
PATHS ${_gflags_roots}
NO_DEFAULT_PATH
PATH_SUFFIXES "lib" )
else()
find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h
# make sure we don't accidentally pick up a different version
NO_CMAKE_SYSTEM_PATH
NO_SYSTEM_ENVIRONMENT_PATH)
find_library(GFLAGS_SHARED_LIB gflags
NO_CMAKE_SYSTEM_PATH
NO_SYSTEM_ENVIRONMENT_PATH)
find_library(GFLAGS_STATIC_LIB libgflags.a
NO_CMAKE_SYSTEM_PATH
NO_SYSTEM_ENVIRONMENT_PATH)
endif()
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(GFLAGS REQUIRED_VARS
GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB GFLAGS_INCLUDE_DIR)
cpp/src/arrow/CMakeLists.txt
+2
View File
@@ -21,6 +21,7 @@ install(FILES
array.h
column.h
builder.h
pretty_print.h
schema.h
table.h
type.h
@@ -37,6 +38,7 @@ set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
ADD_ARROW_TEST(array-test)
ADD_ARROW_TEST(column-test)
ADD_ARROW_TEST(pretty_print-test)
ADD_ARROW_TEST(schema-test)
ADD_ARROW_TEST(table-test)
cpp/src/arrow/array-test.cc
+1
View File
@@ -18,6 +18,7 @@
#include <cstdint>
#include <cstdlib>
#include <memory>
#include <numeric>
#include <vector>
#include "gtest/gtest.h"
cpp/src/arrow/io/CMakeLists.txt
+10 -4
View File
@@ -18,10 +18,16 @@
# ----------------------------------------------------------------------
# arrow_io : Arrow IO interfaces
set(ARROW_IO_LINK_LIBS
arrow_shared
dl
)
if (MSVC)
set(ARROW_IO_LINK_LIBS
arrow_shared
)
else()
set(ARROW_IO_LINK_LIBS
arrow_shared
dl
)
endif()
if (ARROW_BOOST_USE_SHARED)
set(ARROW_IO_PRIVATE_LINK_LIBS
cpp/src/arrow/io/file.cc
+4 -3
View File
@@ -186,12 +186,13 @@ static inline Status FileOpenWriteable(const std::string& filename, int* fd) {
memcpy(wpath.data(), filename.data(), filename.size());
memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
errno_actual = _wsopen_s(
fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY, _SH_DENYNO, _S_IWRITE);
errno_actual = _wsopen_s(fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY | _O_TRUNC,
_SH_DENYNO, _S_IWRITE);
ret = *fd;
#else
ret = *fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY, ARROW_WRITE_SHMODE);
ret = *fd =
open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY | O_TRUNC, ARROW_WRITE_SHMODE);
#endif
return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
}
cpp/src/arrow/io/io-file-test.cc
+7
View File
@@ -18,7 +18,9 @@
#include <cstdint>
#include <cstdio>
#include <cstring>
#ifndef _MSC_VER
#include <fcntl.h>
#endif
#include <fstream>
#include <memory>
#include <sstream>
@@ -38,7 +40,12 @@ static bool FileExists(const std::string& path) {
}
static bool FileIsClosed(int fd) {
#ifdef _MSC_VER
// Close file a second time, this should set errno to EBADF
close(fd);
#else
if (-1 != fcntl(fd, F_GETFD)) { return false; }
#endif
return errno == EBADF;
}
cpp/src/arrow/io/memory.cc
+29 -9
View File
@@ -17,7 +17,18 @@
#include "arrow/io/memory.h"
#include <sys/mman.h> // For memory-mapping
// sys/mman.h not present in Visual Studio or Cygwin
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include "arrow/io/mman.h"
#undef Realloc
#undef Free
#include <windows.h>
#else
#include <sys/mman.h>
#endif
#include <algorithm>
#include <cerrno>
@@ -247,8 +258,11 @@ Status BufferOutputStream::Reserve(int64_t nbytes) {
// ----------------------------------------------------------------------
// In-memory buffer reader
BufferReader::BufferReader(const uint8_t* buffer, int buffer_size)
: buffer_(buffer), buffer_size_(buffer_size), position_(0) {}
BufferReader::BufferReader(const std::shared_ptr<Buffer>& buffer)
: buffer_(buffer), data_(buffer->data()), size_(buffer->size()), position_(0) {}
BufferReader::BufferReader(const uint8_t* data, int64_t size)
: buffer_(nullptr), data_(data), size_(size), position_(0) {}
BufferReader::~BufferReader() {}
@@ -267,26 +281,32 @@ bool BufferReader::supports_zero_copy() const {
}
Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) {
memcpy(buffer, buffer_ + position_, nbytes);
*bytes_read = std::min(nbytes, buffer_size_ - position_);
memcpy(buffer, data_ + position_, nbytes);
*bytes_read = std::min(nbytes, size_ - position_);
position_ += *bytes_read;
return Status::OK();
}
Status BufferReader::Read(int64_t nbytes, std::shared_ptr<Buffer>* out) {
int64_t size = std::min(nbytes, buffer_size_ - position_);
*out = std::make_shared<Buffer>(buffer_ + position_, size);
int64_t size = std::min(nbytes, size_ - position_);
if (buffer_ != nullptr) {
*out = SliceBuffer(buffer_, position_, size);
} else {
*out = std::make_shared<Buffer>(data_ + position_, size);
}
position_ += nbytes;
return Status::OK();
}
Status BufferReader::GetSize(int64_t* size) {
*size = buffer_size_;
*size = size_;
return Status::OK();
}
Status BufferReader::Seek(int64_t position) {
if (position < 0 || position >= buffer_size_) {
if (position < 0 || position >= size_) {
return Status::IOError("position out of bounds");
}
cpp/src/arrow/io/memory.h
+5 -3
View File
@@ -99,7 +99,8 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
class ARROW_EXPORT BufferReader : public ReadableFileInterface {
public:
BufferReader(const uint8_t* buffer, int buffer_size);
explicit BufferReader(const std::shared_ptr<Buffer>& buffer);
BufferReader(const uint8_t* data, int64_t size);
~BufferReader();
Status Close() override;
@@ -116,8 +117,9 @@ class ARROW_EXPORT BufferReader : public ReadableFileInterface {
bool supports_zero_copy() const override;
private:
const uint8_t* buffer_;
int buffer_size_;
std::shared_ptr<Buffer> buffer_;
const uint8_t* data_;
int64_t size_;
int64_t position_;
};
cpp/src/arrow/io/mman.h
+6 -6
View File
@@ -76,7 +76,7 @@ static DWORD __map_mmap_prot_file(const int prot) {
return desiredAccess;
}
void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
static void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
HANDLE fm, h;
void* map = MAP_FAILED;
@@ -143,7 +143,7 @@ void* mmap(void* addr, size_t len, int prot, int flags, int fildes, off_t off) {
return map;
}
int munmap(void* addr, size_t len) {
static int munmap(void* addr, size_t len) {
if (UnmapViewOfFile(addr)) return 0;
errno = __map_mman_error(GetLastError(), EPERM);
@@ -151,7 +151,7 @@ int munmap(void* addr, size_t len) {
return -1;
}
int mprotect(void* addr, size_t len, int prot) {
static int mprotect(void* addr, size_t len, int prot) {
DWORD newProtect = __map_mmap_prot_page(prot);
DWORD oldProtect = 0;
@@ -162,7 +162,7 @@ int mprotect(void* addr, size_t len, int prot) {
return -1;
}
int msync(void* addr, size_t len, int flags) {
static int msync(void* addr, size_t len, int flags) {
if (FlushViewOfFile(addr, len)) return 0;
errno = __map_mman_error(GetLastError(), EPERM);
@@ -170,7 +170,7 @@ int msync(void* addr, size_t len, int flags) {
return -1;
}
int mlock(const void* addr, size_t len) {
static int mlock(const void* addr, size_t len) {
if (VirtualLock((LPVOID)addr, len)) return 0;
errno = __map_mman_error(GetLastError(), EPERM);
@@ -178,7 +178,7 @@ int mlock(const void* addr, size_t len) {
return -1;
}
int munlock(const void* addr, size_t len) {
static int munlock(const void* addr, size_t len) {
if (VirtualUnlock((LPVOID)addr, len)) return 0;
errno = __map_mman_error(GetLastError(), EPERM);
cpp/src/arrow/io/test-common.h
+12
View File
@@ -24,6 +24,14 @@
#include <string>
#include <vector>
#if defined(__MINGW32__) // MinGW
// nothing
#elif defined(_MSC_VER) // Visual Studio
#include <io.h>
#else // POSIX / Linux
// nothing
#endif
#include "arrow/io/memory.h"
#include "arrow/test-util.h"
#include "arrow/util/buffer.h"
@@ -43,7 +51,11 @@ class MemoryMapFixture {
void CreateFile(const std::string path, int64_t size) {
FILE* file = fopen(path.c_str(), "w");
if (file != nullptr) { tmp_files_.push_back(path); }
#ifdef _MSC_VER
_chsize(fileno(file), size);
#else
ftruncate(fileno(file), size);
#endif
fclose(file);
}
cpp/src/arrow/ipc/CMakeLists.txt
+29
View File
@@ -85,6 +85,35 @@ ADD_ARROW_TEST(ipc-json-test)
ARROW_TEST_LINK_LIBRARIES(ipc-json-test
${ARROW_IPC_TEST_LINK_LIBS})
ADD_ARROW_TEST(json-integration-test)
if (ARROW_BUILD_TESTS)
if (APPLE)
target_link_libraries(json-integration-test
arrow_static
arrow_io
arrow_ipc
gflags
gtest
boost_filesystem_static
boost_system_static
dl)
set_target_properties(json-integration-test
PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
else()
target_link_libraries(json-integration-test
arrow_static
arrow_io
arrow_ipc
gflags
gtest
pthread
boost_filesystem_static
boost_system_static
dl)
endif()
endif()
# make clean will delete the generated file
set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
cpp/src/arrow/ipc/adapter.cc
+137 -116
View File
@@ -48,15 +48,6 @@ namespace flatbuf = org::apache::arrow::flatbuf;
namespace ipc {
namespace {
Status CheckMultipleOf64(int64_t size) {
if (BitUtil::IsMultipleOf64(size)) { return Status::OK(); }
return Status::Invalid(
"Attempted to write a buffer that "
"wasn't a multiple of 64 bytes");
}
}
static bool IsPrimitive(const DataType* type) {
DCHECK(type != nullptr);
switch (type->type) {
@@ -124,30 +115,30 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
class RecordBatchWriter {
public:
RecordBatchWriter(const std::vector<std::shared_ptr<Array>>& columns, int32_t num_rows,
int max_recursion_depth)
int64_t buffer_start_offset, int max_recursion_depth)
: columns_(&columns),
num_rows_(num_rows),
buffer_start_offset_(buffer_start_offset),
max_recursion_depth_(max_recursion_depth) {}
Status AssemblePayload() {
Status AssemblePayload(int64_t* body_length) {
if (field_nodes_.size() > 0) {
field_nodes_.clear();
buffer_meta_.clear();
buffers_.clear();
}
// Perform depth-first traversal of the row-batch
for (size_t i = 0; i < columns_->size(); ++i) {
const Array* arr = (*columns_)[i].get();
RETURN_NOT_OK(VisitArray(arr, &field_nodes_, &buffers_, max_recursion_depth_));
}
return Status::OK();
}
Status Write(
io::OutputStream* dst, int64_t* body_end_offset, int64_t* header_end_offset) {
// Get the starting position
int64_t start_position;
RETURN_NOT_OK(dst->Tell(&start_position));
// Keep track of the current position so we can determine the size of the
// message body
int64_t position = start_position;
// The position for the start of a buffer relative to the passed frame of
// reference. May be 0 or some other position in an address space
int64_t offset = buffer_start_offset_;
// Construct the buffer metadata for the record batch header
for (size_t i = 0; i < buffers_.size(); ++i) {
const Buffer* buffer = buffers_[i].get();
int64_t size = 0;
@@ -161,65 +152,103 @@ class RecordBatchWriter {
// TODO(wesm): We currently have no notion of shared memory page id's,
// but we've included it in the metadata IDL for when we have it in the
// future. Use page=0 for now
// future. Use page = -1 for now
//
// Note that page ids are a bespoke notion for Arrow and not a feature we
// are using from any OS-level shared memory. The thought is that systems
// may (in the future) associate integer page id's with physical memory
// pages (according to whatever is the desired shared memory mechanism)
buffer_meta_.push_back(flatbuf::Buffer(0, position, size + padding));
if (size > 0) {
RETURN_NOT_OK(dst->Write(buffer->data(), size));
position += size;
}
if (padding > 0) {
RETURN_NOT_OK(dst->Write(kPaddingBytes, padding));
position += padding;
}
buffer_meta_.push_back(flatbuf::Buffer(-1, offset, size + padding));
offset += size + padding;
}
*body_end_offset = position;
*body_length = offset - buffer_start_offset_;
DCHECK(BitUtil::IsMultipleOf64(*body_length));
return Status::OK();
}
Status WriteMetadata(
int64_t body_length, io::OutputStream* dst, int32_t* metadata_length) {
// Now that we have computed the locations of all of the buffers in shared
// memory, the data header can be converted to a flatbuffer and written out
//
// Note: The memory written here is prefixed by the size of the flatbuffer
// itself as an int32_t. On reading from a input, you will have to
// determine the data header size then request a buffer such that you can
// construct the flatbuffer data accessor object (see arrow::ipc::Message)
std::shared_ptr<Buffer> data_header;
RETURN_NOT_OK(WriteDataHeader(
num_rows_, position - start_position, field_nodes_, buffer_meta_, &data_header));
// itself as an int32_t.
std::shared_ptr<Buffer> metadata_fb;
RETURN_NOT_OK(WriteRecordBatchMetadata(
num_rows_, body_length, field_nodes_, buffer_meta_, &metadata_fb));
// Write the data header at the end
RETURN_NOT_OK(dst->Write(data_header->data(), data_header->size()));
// Need to write 4 bytes (metadata size), the metadata, plus padding to
// fall on a 64-byte offset
int64_t padded_metadata_length =
BitUtil::RoundUpToMultipleOf64(metadata_fb->size() + 4);
position += data_header->size();
*header_end_offset = position;
// The returned metadata size includes the length prefix, the flatbuffer,
// plus padding
*metadata_length = padded_metadata_length;
return Align(dst, &position);
// Write the flatbuffer size prefix
int32_t flatbuffer_size = metadata_fb->size();
RETURN_NOT_OK(
dst->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
// Write the flatbuffer
RETURN_NOT_OK(dst->Write(metadata_fb->data(), metadata_fb->size()));
// Write any padding
int64_t padding = padded_metadata_length - metadata_fb->size() - 4;
if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
return Status::OK();
}
Status Write(io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
RETURN_NOT_OK(AssemblePayload(body_length));
#ifndef NDEBUG
int64_t start_position, current_position;
RETURN_NOT_OK(dst->Tell(&start_position));
#endif
RETURN_NOT_OK(WriteMetadata(*body_length, dst, metadata_length));
#ifndef NDEBUG
RETURN_NOT_OK(dst->Tell(&current_position));
DCHECK(BitUtil::IsMultipleOf8(current_position));
#endif
// Now write the buffers
for (size_t i = 0; i < buffers_.size(); ++i) {
const Buffer* buffer = buffers_[i].get();
int64_t size = 0;
int64_t padding = 0;
// The buffer might be null if we are handling zero row lengths.
if (buffer) {
size = buffer->size();
padding = BitUtil::RoundUpToMultipleOf64(size) - size;
}
if (size > 0) { RETURN_NOT_OK(dst->Write(buffer->data(), size)); }
if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
}
#ifndef NDEBUG
RETURN_NOT_OK(dst->Tell(&current_position));
DCHECK(BitUtil::IsMultipleOf8(current_position));
#endif
Status Align(io::OutputStream* dst, int64_t* position) {
// Write all buffers here on word boundaries
// TODO(wesm): Is there benefit to 64-byte padding in IPC?
int64_t remainder = PaddedLength(*position) - *position;
if (remainder > 0) {
RETURN_NOT_OK(dst->Write(kPaddingBytes, remainder));
*position += remainder;
}
return Status::OK();
}
// This must be called after invoking AssemblePayload
Status GetTotalSize(int64_t* size) {
// emulates the behavior of Write without actually writing
int64_t body_offset;
int64_t data_header_offset;
int32_t metadata_length;
int64_t body_length;
MockOutputStream dst;
RETURN_NOT_OK(Write(&dst, &body_offset, &data_header_offset));
RETURN_NOT_OK(Write(&dst, &metadata_length, &body_length));
*size = dst.GetExtentBytesWritten();
return Status::OK();
}
@@ -228,6 +257,7 @@ class RecordBatchWriter {
// Do not copy this vector. Ownership must be retained elsewhere
const std::vector<std::shared_ptr<Array>>* columns_;
int32_t num_rows_;
int64_t buffer_start_offset_;
std::vector<flatbuf::FieldNode> field_nodes_;
std::vector<flatbuf::Buffer> buffer_meta_;
@@ -236,18 +266,17 @@ class RecordBatchWriter {
};
Status WriteRecordBatch(const std::vector<std::shared_ptr<Array>>& columns,
int32_t num_rows, io::OutputStream* dst, int64_t* body_end_offset,
int64_t* header_end_offset, int max_recursion_depth) {
int32_t num_rows, int64_t buffer_start_offset, io::OutputStream* dst,
int32_t* metadata_length, int64_t* body_length, int max_recursion_depth) {
DCHECK_GT(max_recursion_depth, 0);
RecordBatchWriter serializer(columns, num_rows, max_recursion_depth);
RETURN_NOT_OK(serializer.AssemblePayload());
return serializer.Write(dst, body_end_offset, header_end_offset);
RecordBatchWriter serializer(
columns, num_rows, buffer_start_offset, max_recursion_depth);
return serializer.Write(dst, metadata_length, body_length);
}
Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size) {
RecordBatchWriter serializer(
batch->columns(), batch->num_rows(), kMaxIpcRecursionDepth);
RETURN_NOT_OK(serializer.AssemblePayload());
batch->columns(), batch->num_rows(), 0, kMaxIpcRecursionDepth);
RETURN_NOT_OK(serializer.GetTotalSize(size));
return Status::OK();
}
@@ -255,30 +284,33 @@ Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size) {
// ----------------------------------------------------------------------
// Record batch read path
class RecordBatchReader::RecordBatchReaderImpl {
class RecordBatchReader {
public:
RecordBatchReaderImpl(io::ReadableFileInterface* file,
const std::shared_ptr<RecordBatchMessage>& metadata, int max_recursion_depth)
: file_(file), metadata_(metadata), max_recursion_depth_(max_recursion_depth) {
RecordBatchReader(const std::shared_ptr<RecordBatchMetadata>& metadata,
const std::shared_ptr<Schema>& schema, int max_recursion_depth,
io::ReadableFileInterface* file)
: metadata_(metadata),
schema_(schema),
max_recursion_depth_(max_recursion_depth),
file_(file) {
num_buffers_ = metadata->num_buffers();
num_flattened_fields_ = metadata->num_fields();
}
Status AssembleBatch(
const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out) {
std::vector<std::shared_ptr<Array>> arrays(schema->num_fields());
Status Read(std::shared_ptr<RecordBatch>* out) {
std::vector<std::shared_ptr<Array>> arrays(schema_->num_fields());
// The field_index and buffer_index are incremented in NextArray based on
// how much of the batch is "consumed" (through nested data reconstruction,
// for example)
field_index_ = 0;
buffer_index_ = 0;
for (int i = 0; i < schema->num_fields(); ++i) {
const Field* field = schema->field(i).get();
for (int i = 0; i < schema_->num_fields(); ++i) {
const Field* field = schema_->field(i).get();
RETURN_NOT_OK(NextArray(field, max_recursion_depth_, &arrays[i]));
}
*out = std::make_shared<RecordBatch>(schema, metadata_->length(), arrays);
*out = std::make_shared<RecordBatch>(schema_, metadata_->length(), arrays);
return Status::OK();
}
@@ -370,67 +402,56 @@ class RecordBatchReader::RecordBatchReaderImpl {
Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) {
BufferMetadata metadata = metadata_->buffer(buffer_index);
RETURN_NOT_OK(CheckMultipleOf64(metadata.length));
return file_->ReadAt(metadata.offset, metadata.length, out);
if (metadata.length == 0) {
*out = std::make_shared<Buffer>(nullptr, 0);
return Status::OK();
} else {
return file_->ReadAt(metadata.offset, metadata.length, out);
}
}
private:
std::shared_ptr<RecordBatchMetadata> metadata_;
std::shared_ptr<Schema> schema_;
int max_recursion_depth_;
io::ReadableFileInterface* file_;
std::shared_ptr<RecordBatchMessage> metadata_;
int field_index_;
int buffer_index_;
int max_recursion_depth_;
int num_buffers_;
int num_flattened_fields_;
};
Status RecordBatchReader::Open(io::ReadableFileInterface* file, int64_t offset,
std::shared_ptr<RecordBatchReader>* out) {
return Open(file, offset, kMaxIpcRecursionDepth, out);
}
Status RecordBatchReader::Open(io::ReadableFileInterface* file, int64_t offset,
int max_recursion_depth, std::shared_ptr<RecordBatchReader>* out) {
Status ReadRecordBatchMetadata(int64_t offset, int32_t metadata_length,
io::ReadableFileInterface* file, std::shared_ptr<RecordBatchMetadata>* metadata) {
std::shared_ptr<Buffer> buffer;
RETURN_NOT_OK(file->ReadAt(offset - sizeof(int32_t), sizeof(int32_t), &buffer));
RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
int32_t metadata_size = *reinterpret_cast<const int32_t*>(buffer->data());
int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
if (metadata_size + static_cast<int>(sizeof(int32_t)) > offset) {
return Status::Invalid("metadata size invalid");
if (flatbuffer_size + static_cast<int>(sizeof(int32_t)) > metadata_length) {
std::stringstream ss;
ss << "flatbuffer size " << metadata_length << " invalid. File offset: " << offset
<< ", metadata length: " << metadata_length;
return Status::Invalid(ss.str());
}
// Read the metadata
RETURN_NOT_OK(
file->ReadAt(offset - metadata_size - sizeof(int32_t), metadata_size, &buffer));
// TODO(wesm): buffer slicing here would be better in case ReadAt returns
// allocated memory
std::shared_ptr<Message> message;
RETURN_NOT_OK(Message::Open(buffer, &message));
if (message->type() != Message::RECORD_BATCH) {
return Status::Invalid("Metadata message is not a record batch");
}
std::shared_ptr<RecordBatchMessage> batch_meta = message->GetRecordBatch();
std::shared_ptr<RecordBatchReader> result(new RecordBatchReader());
result->impl_.reset(new RecordBatchReaderImpl(file, batch_meta, max_recursion_depth));
*out = result;
*metadata = std::make_shared<RecordBatchMetadata>(buffer, sizeof(int32_t));
return Status::OK();
}
// Here the explicit destructor is required for compilers to be aware of
// the complete information of RecordBatchReader::RecordBatchReaderImpl class
RecordBatchReader::~RecordBatchReader() {}
Status ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
const std::shared_ptr<Schema>& schema, io::ReadableFileInterface* file,
std::shared_ptr<RecordBatch>* out) {
return ReadRecordBatch(metadata, schema, kMaxIpcRecursionDepth, file, out);
}
Status RecordBatchReader::GetRecordBatch(
const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out) {
return impl_->AssembleBatch(schema, out);
Status ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
const std::shared_ptr<Schema>& schema, int max_recursion_depth,
io::ReadableFileInterface* file, std::shared_ptr<RecordBatch>* out) {
RecordBatchReader reader(metadata, schema, max_recursion_depth, file);
return reader.Read(out);
}
} // namespace ipc
cpp/src/arrow/ipc/adapter.h
+32 -31
View File
@@ -16,7 +16,7 @@
// under the License.
// Public API for writing and accessing (with zero copy, if possible) Arrow
// data in shared memory
// IPC binary formatted data (e.g. in shared memory, or from some other IO source)
#ifndef ARROW_IPC_ADAPTER_H
#define ARROW_IPC_ADAPTER_H
@@ -43,7 +43,7 @@ class OutputStream;
namespace ipc {
class RecordBatchMessage;
class RecordBatchMetadata;
// ----------------------------------------------------------------------
// Write path
@@ -51,22 +51,30 @@ class RecordBatchMessage;
// TODO(emkornfield) investigate this more
constexpr int kMaxIpcRecursionDepth = 64;
// Write the RecordBatch (collection of equal-length Arrow arrays) to the output
// stream
// Write the RecordBatch (collection of equal-length Arrow arrays) to the
// output stream in a contiguous block. The record batch metadata is written as
// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
// prefixed by its size, followed by each of the memory buffers in the batch
// written end to end (with appropriate alignment and padding):
//
// First, each of the memory buffers are written out end-to-end
//
// Then, this function writes the batch metadata as a flatbuffer (see
// format/Message.fbs -- the RecordBatch message type) like so:
//
// <int32: metadata size> <uint8*: metadata>
// <int32: metadata size> <uint8*: metadata> <buffers>
//
// Finally, the absolute offsets (relative to the start of the output stream)
// to the end of the body and end of the metadata / data header (suffixed by
// the header size) is returned in out-variables
//
// @param(in) buffer_start_offset: the start offset to use in the buffer metadata,
// default should be 0
//
// @param(out) metadata_length: the size of the length-prefixed flatbuffer
// including padding to a 64-byte boundary
//
// @param(out) body_length: the size of the contiguous buffer block plus
// padding bytes
ARROW_EXPORT Status WriteRecordBatch(const std::vector<std::shared_ptr<Array>>& columns,
int32_t num_rows, io::OutputStream* dst, int64_t* body_end_offset,
int64_t* header_end_offset, int max_recursion_depth = kMaxIpcRecursionDepth);
int32_t num_rows, int64_t buffer_start_offset, io::OutputStream* dst,
int32_t* metadata_length, int64_t* body_length,
int max_recursion_depth = kMaxIpcRecursionDepth);
// int64_t GetRecordBatchMetadata(const RecordBatch* batch);
@@ -78,27 +86,20 @@ ARROW_EXPORT Status GetRecordBatchSize(const RecordBatch* batch, int64_t* size);
// ----------------------------------------------------------------------
// "Read" path; does not copy data if the input supports zero copy reads
class ARROW_EXPORT RecordBatchReader {
public:
// The offset is the absolute position to the *end* of the record batch data
// header
static Status Open(io::ReadableFileInterface* file, int64_t offset,
std::shared_ptr<RecordBatchReader>* out);
// Read the record batch flatbuffer metadata starting at the indicated file offset
//
// The flatbuffer is expected to be length-prefixed, so the metadata_length
// includes at least the length prefix and the flatbuffer
Status ARROW_EXPORT ReadRecordBatchMetadata(int64_t offset, int32_t metadata_length,
io::ReadableFileInterface* file, std::shared_ptr<RecordBatchMetadata>* metadata);
static Status Open(io::ReadableFileInterface* file, int64_t offset,
int max_recursion_depth, std::shared_ptr<RecordBatchReader>* out);
Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
const std::shared_ptr<Schema>& schema, io::ReadableFileInterface* file,
std::shared_ptr<RecordBatch>* out);
virtual ~RecordBatchReader();
// Reassemble the record batch. A Schema is required to be able to construct
// the right array containers
Status GetRecordBatch(
const std::shared_ptr<Schema>& schema, std::shared_ptr<RecordBatch>* out);
private:
class RecordBatchReaderImpl;
std::unique_ptr<RecordBatchReaderImpl> impl_;
};
Status ARROW_EXPORT ReadRecordBatch(const std::shared_ptr<RecordBatchMetadata>& metadata,
const std::shared_ptr<Schema>& schema, int max_recursion_depth,
io::ReadableFileInterface* file, std::shared_ptr<RecordBatch>* out);
} // namespace ipc
} // namespace arrow
cpp/src/arrow/ipc/file.cc
+19 -12
View File
@@ -23,6 +23,7 @@
#include <vector>
#include "arrow/io/interfaces.h"
#include "arrow/io/memory.h"
#include "arrow/ipc/adapter.h"
#include "arrow/ipc/metadata.h"
#include "arrow/ipc/util.h"
@@ -87,19 +88,19 @@ Status FileWriter::WriteRecordBatch(
int64_t offset = position_;
int64_t body_end_offset;
int64_t header_end_offset;
// There may be padding ever the end of the metadata, so we cannot rely on
// position_
int32_t metadata_length;
int64_t body_length;
// Frame of reference in file format is 0, see ARROW-384
const int64_t buffer_start_offset = 0;
RETURN_NOT_OK(arrow::ipc::WriteRecordBatch(
columns, num_rows, sink_, &body_end_offset, &header_end_offset));
columns, num_rows, buffer_start_offset, sink_, &metadata_length, &body_length));
RETURN_NOT_OK(UpdatePosition());
DCHECK(position_ % 8 == 0) << "ipc::WriteRecordBatch did not perform aligned writes";
// There may be padding ever the end of the metadata, so we cannot rely on
// position_
int32_t metadata_length = header_end_offset - body_end_offset;
int32_t body_length = body_end_offset - offset;
// Append metadata, to be written in the footer later
record_batches_.emplace_back(offset, metadata_length, body_length);
@@ -198,12 +199,18 @@ Status FileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
DCHECK_GE(i, 0);
DCHECK_LT(i, num_record_batches());
FileBlock block = footer_->record_batch(i);
int64_t metadata_end_offset = block.offset + block.body_length + block.metadata_length;
std::shared_ptr<RecordBatchReader> reader;
RETURN_NOT_OK(RecordBatchReader::Open(file_.get(), metadata_end_offset, &reader));
std::shared_ptr<RecordBatchMetadata> metadata;
RETURN_NOT_OK(ReadRecordBatchMetadata(
block.offset, block.metadata_length, file_.get(), &metadata));
return reader->GetRecordBatch(schema_, batch);
// TODO(wesm): ARROW-388 -- the buffer frame of reference is 0 (see
// ARROW-384).
std::shared_ptr<Buffer> buffer_block;
RETURN_NOT_OK(file_->Read(block.body_length, &buffer_block));
io::BufferReader reader(buffer_block);
return ReadRecordBatch(metadata, schema_, &reader, batch);
}
} // namespace ipc
cpp/src/arrow/ipc/ipc-adapter-test.cc
+50 -35
View File
@@ -54,17 +54,24 @@ class TestWriteRecordBatch : public ::testing::TestWithParam<MakeRecordBatch*>,
std::string path = "test-write-row-batch";
io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
int64_t body_end_offset;
int64_t header_end_offset;
int32_t metadata_length;
int64_t body_length;
RETURN_NOT_OK(WriteRecordBatch(batch.columns(), batch.num_rows(), mmap_.get(),
&body_end_offset, &header_end_offset));
const int64_t buffer_offset = 0;
std::shared_ptr<RecordBatchReader> reader;
RETURN_NOT_OK(RecordBatchReader::Open(mmap_.get(), header_end_offset, &reader));
RETURN_NOT_OK(WriteRecordBatch(batch.columns(), batch.num_rows(), buffer_offset,
mmap_.get(), &metadata_length, &body_length));
RETURN_NOT_OK(reader->GetRecordBatch(batch.schema(), batch_result));
return Status::OK();
std::shared_ptr<RecordBatchMetadata> metadata;
RETURN_NOT_OK(ReadRecordBatchMetadata(0, metadata_length, mmap_.get(), &metadata));
// The buffer offsets start at 0, so we must construct a
// ReadableFileInterface according to that frame of reference
std::shared_ptr<Buffer> buffer_payload;
RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
io::BufferReader buffer_reader(buffer_payload);
return ReadRecordBatch(metadata, batch.schema(), &buffer_reader, batch_result);
}
protected:
@@ -96,11 +103,11 @@ INSTANTIATE_TEST_CASE_P(RoundTripTests, TestWriteRecordBatch,
void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
ipc::MockOutputStream mock;
int64_t mock_header_offset = -1;
int64_t mock_body_offset = -1;
int32_t mock_metadata_length = -1;
int64_t mock_body_length = -1;
int64_t size = -1;
ASSERT_OK(WriteRecordBatch(batch->columns(), batch->num_rows(), &mock,
&mock_body_offset, &mock_header_offset));
ASSERT_OK(WriteRecordBatch(batch->columns(), batch->num_rows(), 0, &mock,
&mock_metadata_length, &mock_body_length));
ASSERT_OK(GetRecordBatchSize(batch.get(), &size));
ASSERT_EQ(mock.GetExtentBytesWritten(), size);
}
@@ -129,39 +136,36 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
void SetUp() { pool_ = default_memory_pool(); }
void TearDown() { io::MemoryMapFixture::TearDown(); }
Status WriteToMmap(int recursion_level, bool override_level,
int64_t* header_out = nullptr, std::shared_ptr<Schema>* schema_out = nullptr) {
Status WriteToMmap(int recursion_level, bool override_level, int32_t* metadata_length,
int64_t* body_length, std::shared_ptr<Schema>* schema) {
const int batch_length = 5;
TypePtr type = kInt32;
TypePtr type = int32();
ArrayPtr array;
const bool include_nulls = true;
RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
for (int i = 0; i < recursion_level; ++i) {
type = std::static_pointer_cast<DataType>(std::make_shared<ListType>(type));
type = list(type);
RETURN_NOT_OK(
MakeRandomListArray(array, batch_length, include_nulls, pool_, &array));
}
auto f0 = std::make_shared<Field>("f0", type);
std::shared_ptr<Schema> schema(new Schema({f0}));
if (schema_out != nullptr) { *schema_out = schema; }
auto f0 = field("f0", type);
*schema = std::shared_ptr<Schema>(new Schema({f0}));
std::vector<ArrayPtr> arrays = {array};
auto batch = std::make_shared<RecordBatch>(schema, batch_length, arrays);
auto batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
std::string path = "test-write-past-max-recursion";
const int memory_map_size = 1 << 16;
io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
int64_t body_offset;
int64_t header_offset;
int64_t* header_out_param = header_out == nullptr ? &header_offset : header_out;
if (override_level) {
return WriteRecordBatch(batch->columns(), batch->num_rows(), mmap_.get(),
&body_offset, header_out_param, recursion_level + 1);
return WriteRecordBatch(batch->columns(), batch->num_rows(), 0, mmap_.get(),
metadata_length, body_length, recursion_level + 1);
} else {
return WriteRecordBatch(batch->columns(), batch->num_rows(), mmap_.get(),
&body_offset, header_out_param);
return WriteRecordBatch(batch->columns(), batch->num_rows(), 0, mmap_.get(),
metadata_length, body_length);
}
}
@@ -171,18 +175,29 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
};
TEST_F(RecursionLimits, WriteLimit) {
ASSERT_RAISES(Invalid, WriteToMmap((1 << 8) + 1, false));
int32_t metadata_length = -1;
int64_t body_length = -1;
std::shared_ptr<Schema> schema;
ASSERT_RAISES(
Invalid, WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length, &schema));
}
TEST_F(RecursionLimits, ReadLimit) {
int64_t header_offset = -1;
int32_t metadata_length = -1;
int64_t body_length = -1;
std::shared_ptr<Schema> schema;
ASSERT_OK(WriteToMmap(64, true, &header_offset, &schema));
ASSERT_OK(WriteToMmap(64, true, &metadata_length, &body_length, &schema));
std::shared_ptr<RecordBatchReader> reader;
ASSERT_OK(RecordBatchReader::Open(mmap_.get(), header_offset, &reader));
std::shared_ptr<RecordBatch> batch_result;
ASSERT_RAISES(Invalid, reader->GetRecordBatch(schema, &batch_result));
std::shared_ptr<RecordBatchMetadata> metadata;
ASSERT_OK(ReadRecordBatchMetadata(0, metadata_length, mmap_.get(), &metadata));
std::shared_ptr<Buffer> payload;
ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
io::BufferReader reader(payload);
std::shared_ptr<RecordBatch> batch;
ASSERT_RAISES(Invalid, ReadRecordBatch(metadata, schema, &reader, &batch));
}
} // namespace ipc
cpp/src/arrow/ipc/ipc-file-test.cc
+1 -1
View File
@@ -68,7 +68,7 @@ class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
RETURN_NOT_OK(sink_->Tell(&footer_offset));
// Open the file
auto reader = std::make_shared<io::BufferReader>(buffer_->data(), buffer_->size());
auto reader = std::make_shared<io::BufferReader>(buffer_);
RETURN_NOT_OK(FileReader::Open(reader, footer_offset, &file_reader_));
EXPECT_EQ(num_batches, file_reader_->num_record_batches());
cpp/src/arrow/ipc/ipc-json-test.cc
+12 -28
View File
@@ -96,26 +96,6 @@ void CheckPrimitive(const std::shared_ptr<DataType>& type,
TestArrayRoundTrip(*array.get());
}
template <typename TYPE, typename C_TYPE>
void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& is_valid,
const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
std::shared_ptr<Buffer> values_buffer;
std::shared_ptr<Buffer> values_bitmap;
ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer));
ASSERT_OK(test::GetBitmapFromBoolVector(is_valid, &values_bitmap));
using ArrayType = typename TypeTraits<TYPE>::ArrayType;
int32_t null_count = 0;
for (bool val : is_valid) {
if (!val) { ++null_count; }
}
*out = std::make_shared<ArrayType>(type, static_cast<int32_t>(values.size()),
values_buffer, null_count, values_bitmap);
}
TEST(TestJsonSchemaWriter, FlatTypes) {
std::vector<std::shared_ptr<Field>> fields = {field("f0", int8()),
field("f1", int16(), false), field("f2", int32()), field("f3", int64(), false),
@@ -284,19 +264,23 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
"name": "foo",
"type": {"name": "int", "isSigned": true, "bitWidth": 32},
"nullable": true, "children": [],
"typeLayout": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "DATA", "typeBitWidth": 32}
]
"typeLayout": {
"vectors": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "DATA", "typeBitWidth": 32}
]
}
},
{
"name": "bar",
"type": {"name": "floatingpoint", "precision": "DOUBLE"},
"nullable": true, "children": [],
"typeLayout": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "DATA", "typeBitWidth": 64}
]
"typeLayout": {
"vectors": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "DATA", "typeBitWidth": 64}
]
}
}
]
},
cpp/src/arrow/ipc/ipc-metadata-test.cc
+6 -6
View File
@@ -43,7 +43,7 @@ static inline void assert_schema_equal(const Schema* lhs, const Schema* rhs) {
}
}
class TestSchemaMessage : public ::testing::Test {
class TestSchemaMetadata : public ::testing::Test {
public:
void SetUp() {}
@@ -52,11 +52,11 @@ class TestSchemaMessage : public ::testing::Test {
ASSERT_OK(WriteSchema(schema, &buffer));
std::shared_ptr<Message> message;
ASSERT_OK(Message::Open(buffer, &message));
ASSERT_OK(Message::Open(buffer, 0, &message));
ASSERT_EQ(Message::SCHEMA, message->type());
std::shared_ptr<SchemaMessage> schema_msg = message->GetSchema();
auto schema_msg = std::make_shared<SchemaMetadata>(message);
ASSERT_EQ(schema->num_fields(), schema_msg->num_fields());
std::shared_ptr<Schema> schema2;
@@ -68,7 +68,7 @@ class TestSchemaMessage : public ::testing::Test {
const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
TEST_F(TestSchemaMessage, PrimitiveFields) {
TEST_F(TestSchemaMetadata, PrimitiveFields) {
auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>());
auto f2 = std::make_shared<Field>("f2", std::make_shared<Int32Type>());
@@ -85,7 +85,7 @@ TEST_F(TestSchemaMessage, PrimitiveFields) {
CheckRoundtrip(&schema);
}
TEST_F(TestSchemaMessage, NestedFields) {
TEST_F(TestSchemaMetadata, NestedFields) {
auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
auto f0 = std::make_shared<Field>("f0", type);
@@ -111,7 +111,7 @@ class TestFileFooter : public ::testing::Test {
std::unique_ptr<FileFooter> footer;
ASSERT_OK(FileFooter::Open(buffer, &footer));
ASSERT_EQ(MetadataVersion::V1_SNAPSHOT, footer->version());
ASSERT_EQ(MetadataVersion::V2, footer->version());
// Check schema
std::shared_ptr<Schema> schema2;
cpp/src/arrow/ipc/json-integration-test.cc
+394
View File
@@ -0,0 +1,394 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <memory>
#include <string>
#include "gflags/gflags.h"
#include "gtest/gtest.h"
#include <boost/filesystem.hpp> // NOLINT
#include "arrow/io/file.h"
#include "arrow/ipc/file.h"
#include "arrow/ipc/json.h"
#include "arrow/pretty_print.h"
#include "arrow/schema.h"
#include "arrow/table.h"
#include "arrow/test-util.h"
#include "arrow/util/status.h"
DEFINE_string(arrow, "", "Arrow file name");
DEFINE_string(json, "", "JSON file name");
DEFINE_string(mode, "VALIDATE",
"Mode of integration testing tool (ARROW_TO_JSON, JSON_TO_ARROW, VALIDATE)");
DEFINE_bool(integration, false, "Run in integration test mode");
DEFINE_bool(verbose, true, "Verbose output");
namespace fs = boost::filesystem;
namespace arrow {
bool file_exists(const char* path) {
std::ifstream handle(path);
return handle.good();
}
// Convert JSON file to IPC binary format
static Status ConvertJsonToArrow(
const std::string& json_path, const std::string& arrow_path) {
std::shared_ptr<io::ReadableFile> in_file;
std::shared_ptr<io::FileOutputStream> out_file;
RETURN_NOT_OK(io::ReadableFile::Open(json_path, &in_file));
RETURN_NOT_OK(io::FileOutputStream::Open(arrow_path, &out_file));
int64_t file_size = 0;
RETURN_NOT_OK(in_file->GetSize(&file_size));
std::shared_ptr<Buffer> json_buffer;
RETURN_NOT_OK(in_file->Read(file_size, &json_buffer));
std::unique_ptr<ipc::JsonReader> reader;
RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &reader));
if (FLAGS_verbose) {
std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
}
std::shared_ptr<ipc::FileWriter> writer;
RETURN_NOT_OK(ipc::FileWriter::Open(out_file.get(), reader->schema(), &writer));
for (int i = 0; i < reader->num_record_batches(); ++i) {
std::shared_ptr<RecordBatch> batch;
RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
RETURN_NOT_OK(writer->WriteRecordBatch(batch->columns(), batch->num_rows()));
}
return writer->Close();
}
// Convert IPC binary format to JSON
static Status ConvertArrowToJson(
const std::string& arrow_path, const std::string& json_path) {
std::shared_ptr<io::ReadableFile> in_file;
std::shared_ptr<io::FileOutputStream> out_file;
RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &in_file));
RETURN_NOT_OK(io::FileOutputStream::Open(json_path, &out_file));
std::shared_ptr<ipc::FileReader> reader;
RETURN_NOT_OK(ipc::FileReader::Open(in_file, &reader));
if (FLAGS_verbose) {
std::cout << "Found schema: " << reader->schema()->ToString() << std::endl;
}
std::unique_ptr<ipc::JsonWriter> writer;
RETURN_NOT_OK(ipc::JsonWriter::Open(reader->schema(), &writer));
for (int i = 0; i < reader->num_record_batches(); ++i) {
std::shared_ptr<RecordBatch> batch;
RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
RETURN_NOT_OK(writer->WriteRecordBatch(batch->columns(), batch->num_rows()));
}
std::string result;
RETURN_NOT_OK(writer->Finish(&result));
return out_file->Write(reinterpret_cast<const uint8_t*>(result.c_str()),
static_cast<int64_t>(result.size()));
}
static Status ValidateArrowVsJson(
const std::string& arrow_path, const std::string& json_path) {
// Construct JSON reader
std::shared_ptr<io::ReadableFile> json_file;
RETURN_NOT_OK(io::ReadableFile::Open(json_path, &json_file));
int64_t file_size = 0;
RETURN_NOT_OK(json_file->GetSize(&file_size));
std::shared_ptr<Buffer> json_buffer;
RETURN_NOT_OK(json_file->Read(file_size, &json_buffer));
std::unique_ptr<ipc::JsonReader> json_reader;
RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &json_reader));
// Construct Arrow reader
std::shared_ptr<io::ReadableFile> arrow_file;
RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &arrow_file));
std::shared_ptr<ipc::FileReader> arrow_reader;
RETURN_NOT_OK(ipc::FileReader::Open(arrow_file, &arrow_reader));
auto json_schema = json_reader->schema();
auto arrow_schema = arrow_reader->schema();
if (!json_schema->Equals(arrow_schema)) {
std::stringstream ss;
ss << "JSON schema: \n"
<< json_schema->ToString() << "\n"
<< "Arrow schema: \n"
<< arrow_schema->ToString();
if (FLAGS_verbose) { std::cout << ss.str() << std::endl; }
return Status::Invalid("Schemas did not match");
}
const int json_nbatches = json_reader->num_record_batches();
const int arrow_nbatches = arrow_reader->num_record_batches();
if (json_nbatches != arrow_nbatches) {
std::stringstream ss;
ss << "Different number of record batches: " << json_nbatches << " (JSON) vs "
<< arrow_nbatches << " (Arrow)";
return Status::Invalid(ss.str());
}
std::shared_ptr<RecordBatch> arrow_batch;
std::shared_ptr<RecordBatch> json_batch;
for (int i = 0; i < json_nbatches; ++i) {
RETURN_NOT_OK(json_reader->GetRecordBatch(i, &json_batch));
RETURN_NOT_OK(arrow_reader->GetRecordBatch(i, &arrow_batch));
if (!json_batch->Equals(*arrow_batch.get())) {
std::stringstream ss;
ss << "Record batch " << i << " did not match";
ss << "\nJSON: \n ";
RETURN_NOT_OK(PrettyPrint(*json_batch.get(), &ss));
ss << "\nArrow: \n ";
RETURN_NOT_OK(PrettyPrint(*arrow_batch.get(), &ss));
return Status::Invalid(ss.str());
}
}
return Status::OK();
}
Status RunCommand(const std::string& json_path, const std::string& arrow_path,
const std::string& command) {
if (json_path == "") { return Status::Invalid("Must specify json file name"); }
if (arrow_path == "") { return Status::Invalid("Must specify arrow file name"); }
if (command == "ARROW_TO_JSON") {
if (!file_exists(arrow_path.c_str())) {
return Status::Invalid("Input file does not exist");
}
return ConvertArrowToJson(arrow_path, json_path);
} else if (command == "JSON_TO_ARROW") {
if (!file_exists(json_path.c_str())) {
return Status::Invalid("Input file does not exist");
}
return ConvertJsonToArrow(json_path, arrow_path);
} else if (command == "VALIDATE") {
if (!file_exists(json_path.c_str())) {
return Status::Invalid("JSON file does not exist");
}
if (!file_exists(arrow_path.c_str())) {
return Status::Invalid("Arrow file does not exist");
}
return ValidateArrowVsJson(arrow_path, json_path);
} else {
std::stringstream ss;
ss << "Unknown command: " << command;
return Status::Invalid(ss.str());
}
}
static std::string temp_path() {
return (fs::temp_directory_path() / fs::unique_path()).native();
}
class TestJSONIntegration : public ::testing::Test {
public:
void SetUp() {}
std::string mkstemp() {
auto path = temp_path();
tmp_paths_.push_back(path);
return path;
}
Status WriteJson(const char* data, const std::string& path) {
do {
std::shared_ptr<io::FileOutputStream> out;
RETURN_NOT_OK(io::FileOutputStream::Open(path, &out));
RETURN_NOT_OK(out->Write(
reinterpret_cast<const uint8_t*>(data), static_cast<int64_t>(strlen(data))));
} while (0);
return Status::OK();
}
void TearDown() {
for (const std::string path : tmp_paths_) {
std::remove(path.c_str());
}
}
protected:
std::vector<std::string> tmp_paths_;
};
static const char* JSON_EXAMPLE = R"example(
{
"schema": {
"fields": [
{
"name": "foo",
"type": {"name": "int", "isSigned": true, "bitWidth": 32},
"nullable": true, "children": [],
"typeLayout": {
"vectors": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "DATA", "typeBitWidth": 32}
]
}
},
{
"name": "bar",
"type": {"name": "floatingpoint", "precision": "DOUBLE"},
"nullable": true, "children": [],
"typeLayout": {
"vectors": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "DATA", "typeBitWidth": 64}
]
}
}
]
},
"batches": [
{
"count": 5,
"columns": [
{
"name": "foo",
"count": 5,
"DATA": [1, 2, 3, 4, 5],
"VALIDITY": [1, 0, 1, 1, 1]
},
{
"name": "bar",
"count": 5,
"DATA": [1.0, 2.0, 3.0, 4.0, 5.0],
"VALIDITY": [1, 0, 0, 1, 1]
}
]
}
]
}
)example";
static const char* JSON_EXAMPLE2 = R"example(
{
"schema": {
"fields": [
{
"name": "foo",
"type": {"name": "int", "isSigned": true, "bitWidth": 32},
"nullable": true, "children": [],
"typeLayout": {
"vectors": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "DATA", "typeBitWidth": 32}
]
}
}
]
},
"batches": [
{
"count": 5,
"columns": [
{
"name": "foo",
"count": 5,
"DATA": [1, 2, 3, 4, 5],
"VALIDITY": [1, 0, 1, 1, 1]
}
]
}
]
}
)example";
TEST_F(TestJSONIntegration, ConvertAndValidate) {
std::string json_path = this->mkstemp();
std::string arrow_path = this->mkstemp();
ASSERT_OK(WriteJson(JSON_EXAMPLE, json_path));
ASSERT_OK(RunCommand(json_path, arrow_path, "JSON_TO_ARROW"));
ASSERT_OK(RunCommand(json_path, arrow_path, "VALIDATE"));
// Convert and overwrite
ASSERT_OK(RunCommand(json_path, arrow_path, "ARROW_TO_JSON"));
// Convert back to arrow, and validate
ASSERT_OK(RunCommand(json_path, arrow_path, "JSON_TO_ARROW"));
ASSERT_OK(RunCommand(json_path, arrow_path, "VALIDATE"));
}
TEST_F(TestJSONIntegration, ErrorStates) {
std::string json_path = this->mkstemp();
std::string json_path2 = this->mkstemp();
std::string arrow_path = this->mkstemp();
ASSERT_OK(WriteJson(JSON_EXAMPLE, json_path));
ASSERT_OK(WriteJson(JSON_EXAMPLE2, json_path2));
ASSERT_OK(ConvertJsonToArrow(json_path, arrow_path));
ASSERT_RAISES(Invalid, ValidateArrowVsJson(arrow_path, json_path2));
ASSERT_RAISES(IOError, ValidateArrowVsJson("does_not_exist-1234", json_path2));
ASSERT_RAISES(IOError, ValidateArrowVsJson(arrow_path, "does_not_exist-1234"));
ASSERT_RAISES(Invalid, RunCommand("", arrow_path, "VALIDATE"));
ASSERT_RAISES(Invalid, RunCommand(json_path, "", "VALIDATE"));
}
} // namespace arrow
int main(int argc, char** argv) {
gflags::ParseCommandLineFlags(&argc, &argv, true);
int ret = 0;
if (FLAGS_integration) {
arrow::Status result = arrow::RunCommand(FLAGS_json, FLAGS_arrow, FLAGS_mode);
if (!result.ok()) {
std::cout << "Error message: " << result.ToString() << std::endl;
ret = 1;
}
} else {
::testing::InitGoogleTest(&argc, argv);
ret = RUN_ALL_TESTS();
}
gflags::ShutDownCommandLineFlags();
return ret;
}
cpp/src/arrow/ipc/json-internal.cc
+32 -80
View File
@@ -45,8 +45,6 @@ namespace ipc {
using RjArray = rj::Value::ConstArray;
using RjObject = rj::Value::ConstObject;
enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY };
static std::string GetBufferTypeName(BufferType type) {
switch (type) {
case BufferType::DATA:
@@ -93,27 +91,6 @@ static std::string GetTimeUnitName(TimeUnit unit) {
return "UNKNOWN";
}
class BufferLayout {
public:
BufferLayout(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {}
BufferType type() const { return type_; }
int bit_width() const { return bit_width_; }
private:
BufferType type_;
int bit_width_;
};
static const BufferLayout kValidityBuffer(BufferType::VALIDITY, 1);
static const BufferLayout kOffsetBuffer(BufferType::OFFSET, 32);
static const BufferLayout kTypeBuffer(BufferType::TYPE, 32);
static const BufferLayout kBooleanBuffer(BufferType::DATA, 1);
static const BufferLayout kValues64(BufferType::DATA, 64);
static const BufferLayout kValues32(BufferType::DATA, 32);
static const BufferLayout kValues16(BufferType::DATA, 16);
static const BufferLayout kValues8(BufferType::DATA, 8);
class JsonSchemaWriter : public TypeVisitor {
public:
explicit JsonSchemaWriter(const Schema& schema, RjWriter* writer)
@@ -154,9 +131,9 @@ class JsonSchemaWriter : public TypeVisitor {
}
template <typename T>
typename std::enable_if<std::is_base_of<NoExtraMeta, T>::value ||
std::is_base_of<BooleanType, T>::value ||
std::is_base_of<NullType, T>::value,
typename std::enable_if<
std::is_base_of<NoExtraMeta, T>::value || std::is_base_of<BooleanType, T>::value ||
std::is_base_of<DateType, T>::value || std::is_base_of<NullType, T>::value,
void>::type
WriteTypeMetadata(const T& type) {}
@@ -243,11 +220,10 @@ class JsonSchemaWriter : public TypeVisitor {
}
template <typename T>
Status WritePrimitive(const std::string& typeclass, const T& type,
const std::vector<BufferLayout>& buffer_layout) {
Status WritePrimitive(const std::string& typeclass, const T& type) {
WriteName(typeclass, type);
SetNoChildren();
WriteBufferLayout(buffer_layout);
WriteBufferLayout(type.GetBufferLayout());
return Status::OK();
}
@@ -255,15 +231,17 @@ class JsonSchemaWriter : public TypeVisitor {
Status WriteVarBytes(const std::string& typeclass, const T& type) {
WriteName(typeclass, type);
SetNoChildren();
WriteBufferLayout({kValidityBuffer, kOffsetBuffer, kValues8});
WriteBufferLayout(type.GetBufferLayout());
return Status::OK();
}
void WriteBufferLayout(const std::vector<BufferLayout>& buffer_layout) {
void WriteBufferLayout(const std::vector<BufferDescr>& buffer_layout) {
writer_->Key("typeLayout");
writer_->StartObject();
writer_->Key("vectors");
writer_->StartArray();
for (const BufferLayout& buffer : buffer_layout) {
for (const BufferDescr& buffer : buffer_layout) {
writer_->StartObject();
writer_->Key("type");
writer_->String(GetBufferTypeName(buffer.type()));
@@ -274,6 +252,7 @@ class JsonSchemaWriter : public TypeVisitor {
writer_->EndObject();
}
writer_->EndArray();
writer_->EndObject();
}
Status WriteChildren(const std::vector<std::shared_ptr<Field>>& children) {
@@ -286,74 +265,52 @@ class JsonSchemaWriter : public TypeVisitor {
return Status::OK();
}
Status Visit(const NullType& type) override { return WritePrimitive("null", type, {}); }
Status Visit(const NullType& type) override { return WritePrimitive("null", type); }
Status Visit(const BooleanType& type) override {
return WritePrimitive("bool", type, {kValidityBuffer, kBooleanBuffer});
}
Status Visit(const BooleanType& type) override { return WritePrimitive("bool", type); }
Status Visit(const Int8Type& type) override {
return WritePrimitive("int", type, {kValidityBuffer, kValues8});
}
Status Visit(const Int8Type& type) override { return WritePrimitive("int", type); }
Status Visit(const Int16Type& type) override {
return WritePrimitive("int", type, {kValidityBuffer, kValues16});
}
Status Visit(const Int16Type& type) override { return WritePrimitive("int", type); }
Status Visit(const Int32Type& type) override {
return WritePrimitive("int", type, {kValidityBuffer, kValues32});
}
Status Visit(const Int32Type& type) override { return WritePrimitive("int", type); }
Status Visit(const Int64Type& type) override {
return WritePrimitive("int", type, {kValidityBuffer, kValues64});
}
Status Visit(const Int64Type& type) override { return WritePrimitive("int", type); }
Status Visit(const UInt8Type& type) override {
return WritePrimitive("int", type, {kValidityBuffer, kValues8});
}
Status Visit(const UInt8Type& type) override { return WritePrimitive("int", type); }
Status Visit(const UInt16Type& type) override {
return WritePrimitive("int", type, {kValidityBuffer, kValues16});
}
Status Visit(const UInt16Type& type) override { return WritePrimitive("int", type); }
Status Visit(const UInt32Type& type) override {
return WritePrimitive("int", type, {kValidityBuffer, kValues32});
}
Status Visit(const UInt32Type& type) override { return WritePrimitive("int", type); }
Status Visit(const UInt64Type& type) override {
return WritePrimitive("int", type, {kValidityBuffer, kValues64});
}
Status Visit(const UInt64Type& type) override { return WritePrimitive("int", type); }
Status Visit(const HalfFloatType& type) override {
return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues16});
return WritePrimitive("floatingpoint", type);
}
Status Visit(const FloatType& type) override {
return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues32});
return WritePrimitive("floatingpoint", type);
}
Status Visit(const DoubleType& type) override {
return WritePrimitive("floatingpoint", type, {kValidityBuffer, kValues64});
return WritePrimitive("floatingpoint", type);
}
Status Visit(const StringType& type) override { return WriteVarBytes("utf8", type); }
Status Visit(const BinaryType& type) override { return WriteVarBytes("binary", type); }
Status Visit(const DateType& type) override {
return WritePrimitive("date", type, {kValidityBuffer, kValues64});
}
Status Visit(const DateType& type) override { return WritePrimitive("date", type); }
Status Visit(const TimeType& type) override {
return WritePrimitive("time", type, {kValidityBuffer, kValues64});
}
Status Visit(const TimeType& type) override { return WritePrimitive("time", type); }
Status Visit(const TimestampType& type) override {
return WritePrimitive("timestamp", type, {kValidityBuffer, kValues64});
return WritePrimitive("timestamp", type);
}
Status Visit(const IntervalType& type) override {
return WritePrimitive("interval", type, {kValidityBuffer, kValues64});
return WritePrimitive("interval", type);
}
Status Visit(const DecimalType& type) override { return Status::NotImplemented("NYI"); }
@@ -361,26 +318,21 @@ class JsonSchemaWriter : public TypeVisitor {
Status Visit(const ListType& type) override {
WriteName("list", type);
RETURN_NOT_OK(WriteChildren(type.children()));
WriteBufferLayout({kValidityBuffer, kOffsetBuffer});
WriteBufferLayout(type.GetBufferLayout());
return Status::OK();
}
Status Visit(const StructType& type) override {
WriteName("struct", type);
WriteChildren(type.children());
WriteBufferLayout({kValidityBuffer, kTypeBuffer});
WriteBufferLayout(type.GetBufferLayout());
return Status::OK();
}
Status Visit(const UnionType& type) override {
WriteName("union", type);
WriteChildren(type.children());
if (type.mode == UnionMode::SPARSE) {
WriteBufferLayout({kValidityBuffer, kTypeBuffer});
} else {
WriteBufferLayout({kValidityBuffer, kTypeBuffer, kOffsetBuffer});
}
WriteBufferLayout(type.GetBufferLayout());
return Status::OK();
}
@@ -391,7 +343,7 @@ class JsonSchemaWriter : public TypeVisitor {
class JsonArrayWriter : public ArrayVisitor {
public:
explicit JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
: name_(name), array_(array), writer_(writer) {}
Status Write() { return VisitArray(name_, array_); }
cpp/src/arrow/ipc/metadata-internal.cc
+63 -37
View File
@@ -37,20 +37,6 @@ namespace flatbuf = org::apache::arrow::flatbuf;
namespace ipc {
const std::shared_ptr<DataType> BOOL = std::make_shared<BooleanType>();
const std::shared_ptr<DataType> INT8 = std::make_shared<Int8Type>();
const std::shared_ptr<DataType> INT16 = std::make_shared<Int16Type>();
const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
const std::shared_ptr<DataType> INT64 = std::make_shared<Int64Type>();
const std::shared_ptr<DataType> UINT8 = std::make_shared<UInt8Type>();
const std::shared_ptr<DataType> UINT16 = std::make_shared<UInt16Type>();
const std::shared_ptr<DataType> UINT32 = std::make_shared<UInt32Type>();
const std::shared_ptr<DataType> UINT64 = std::make_shared<UInt64Type>();
const std::shared_ptr<DataType> FLOAT = std::make_shared<FloatType>();
const std::shared_ptr<DataType> DOUBLE = std::make_shared<DoubleType>();
const std::shared_ptr<DataType> STRING = std::make_shared<StringType>();
const std::shared_ptr<DataType> BINARY = std::make_shared<BinaryType>();
static Status IntFromFlatbuffer(
const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
if (int_data->bitWidth() > 64) {
@@ -62,16 +48,16 @@ static Status IntFromFlatbuffer(
switch (int_data->bitWidth()) {
case 8:
*out = int_data->is_signed() ? INT8 : UINT8;
*out = int_data->is_signed() ? int8() : uint8();
break;
case 16:
*out = int_data->is_signed() ? INT16 : UINT16;
*out = int_data->is_signed() ? int16() : uint16();
break;
case 32:
*out = int_data->is_signed() ? INT32 : UINT32;
*out = int_data->is_signed() ? int32() : uint32();
break;
case 64:
*out = int_data->is_signed() ? INT64 : UINT64;
*out = int_data->is_signed() ? int64() : uint64();
break;
default:
return Status::NotImplemented("Integers not in cstdint are not implemented");
@@ -81,10 +67,12 @@ static Status IntFromFlatbuffer(
static Status FloatFromFlatuffer(
const flatbuf::FloatingPoint* float_data, std::shared_ptr<DataType>* out) {
if (float_data->precision() == flatbuf::Precision_SINGLE) {
*out = FLOAT;
if (float_data->precision() == flatbuf::Precision_HALF) {
*out = float16();
} else if (float_data->precision() == flatbuf::Precision_SINGLE) {
*out = float32();
} else {
*out = DOUBLE;
*out = float64();
}
return Status::OK();
}
@@ -100,13 +88,13 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
return FloatFromFlatuffer(
static_cast<const flatbuf::FloatingPoint*>(type_data), out);
case flatbuf::Type_Binary:
*out = BINARY;
*out = binary();
return Status::OK();
case flatbuf::Type_Utf8:
*out = STRING;
*out = utf8();
return Status::OK();
case flatbuf::Type_Bool:
*out = BOOL;
*out = boolean();
return Status::OK();
case flatbuf::Type_Decimal:
case flatbuf::Type_Timestamp:
@@ -164,7 +152,32 @@ static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type
break;
static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
std::vector<FieldOffset>* children, flatbuf::Type* out_type, Offset* offset) {
std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
flatbuf::Type* out_type, Offset* offset) {
std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
for (const BufferDescr& descr : buffer_layout) {
flatbuf::VectorType vector_type;
switch (descr.type()) {
case BufferType::OFFSET:
vector_type = flatbuf::VectorType_OFFSET;
break;
case BufferType::DATA:
vector_type = flatbuf::VectorType_DATA;
break;
case BufferType::VALIDITY:
vector_type = flatbuf::VectorType_VALIDITY;
break;
case BufferType::TYPE:
vector_type = flatbuf::VectorType_TYPE;
break;
default:
vector_type = flatbuf::VectorType_DATA;
break;
}
auto offset = flatbuf::CreateVectorLayout(fbb, descr.bit_width(), vector_type);
layout->push_back(offset);
}
switch (type->type) {
case Type::BOOL:
*out_type = flatbuf::Type_Bool;
@@ -223,14 +236,18 @@ static Status FieldToFlatbuffer(
flatbuf::Type type_enum;
Offset type_data;
Offset type_layout;
std::vector<FieldOffset> children;
std::vector<VectorLayoutOffset> layout;
RETURN_NOT_OK(TypeToFlatbuffer(fbb, field->type, &children, &type_enum, &type_data));
RETURN_NOT_OK(
TypeToFlatbuffer(fbb, field->type, &children, &layout, &type_enum, &type_data));
auto fb_children = fbb.CreateVector(children);
auto fb_layout = fbb.CreateVector(layout);
// TODO: produce the list of VectorTypes
*offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_data,
field->dictionary, fb_children);
field->dictionary, fb_children, fb_layout);
return Status::OK();
}
@@ -300,13 +317,26 @@ Status MessageBuilder::SetRecordBatch(int32_t length, int64_t body_length,
return Status::OK();
}
Status WriteDataHeader(int32_t length, int64_t body_length,
Status WriteRecordBatchMetadata(int32_t length, int64_t body_length,
const std::vector<flatbuf::FieldNode>& nodes,
const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
MessageBuilder message;
RETURN_NOT_OK(message.SetRecordBatch(length, body_length, nodes, buffers));
RETURN_NOT_OK(message.Finish());
return message.GetBuffer(out);
flatbuffers::FlatBufferBuilder fbb;
auto batch = flatbuf::CreateRecordBatch(
fbb, length, fbb.CreateVectorOfStructs(nodes), fbb.CreateVectorOfStructs(buffers));
fbb.Finish(batch);
int32_t size = fbb.GetSize();
auto result = std::make_shared<PoolBuffer>();
RETURN_NOT_OK(result->Resize(size));
uint8_t* dst = result->mutable_data();
memcpy(dst, fbb.GetBufferPointer(), size);
*out = result;
return Status::OK();
}
Status MessageBuilder::Finish() {
@@ -317,17 +347,13 @@ Status MessageBuilder::Finish() {
}
Status MessageBuilder::GetBuffer(std::shared_ptr<Buffer>* out) {
// The message buffer is suffixed by the size of the complete flatbuffer as
// int32_t
// <uint8_t*: flatbuffer data><int32_t: flatbuffer size>
int32_t size = fbb_.GetSize();
auto result = std::make_shared<PoolBuffer>();
RETURN_NOT_OK(result->Resize(size + sizeof(int32_t)));
RETURN_NOT_OK(result->Resize(size));
uint8_t* dst = result->mutable_data();
memcpy(dst, fbb_.GetBufferPointer(), size);
memcpy(dst + size, reinterpret_cast<int32_t*>(&size), sizeof(int32_t));
*out = result;
return Status::OK();
cpp/src/arrow/ipc/metadata-internal.h
+3 -3
View File
@@ -41,10 +41,10 @@ namespace ipc {
using FBB = flatbuffers::FlatBufferBuilder;
using FieldOffset = flatbuffers::Offset<arrow::flatbuf::Field>;
using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
using Offset = flatbuffers::Offset<void>;
static constexpr flatbuf::MetadataVersion kMetadataVersion =
flatbuf::MetadataVersion_V1_SNAPSHOT;
static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V2;
Status FieldFromFlatbuffer(const flatbuf::Field* field, std::shared_ptr<Field>* out);
@@ -70,7 +70,7 @@ class MessageBuilder {
flatbuffers::FlatBufferBuilder fbb_;
};
Status WriteDataHeader(int32_t length, int64_t body_length,
Status WriteRecordBatchMetadata(int32_t length, int64_t body_length,
const std::vector<flatbuf::FieldNode>& nodes,
const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out);
cpp/src/arrow/ipc/metadata.cc
+65 -50
View File
@@ -50,9 +50,15 @@ Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out) {
class Message::MessageImpl {
public:
explicit MessageImpl(
const std::shared_ptr<Buffer>& buffer, const flatbuf::Message* message)
: buffer_(buffer), message_(message) {}
explicit MessageImpl(const std::shared_ptr<Buffer>& buffer, int64_t offset)
: buffer_(buffer), offset_(offset), message_(nullptr) {}
Status Open() {
message_ = flatbuf::GetMessage(buffer_->data() + offset_);
// TODO(wesm): verify the message
return Status::OK();
}
Message::Type type() const {
switch (message_->header_type()) {
@@ -72,25 +78,23 @@ class Message::MessageImpl {
int64_t body_length() const { return message_->bodyLength(); }
private:
// Owns the memory this message accesses
// Retain reference to memory
std::shared_ptr<Buffer> buffer_;
int64_t offset_;
const flatbuf::Message* message_;
};
Message::Message() {}
Message::Message(const std::shared_ptr<Buffer>& buffer, int64_t offset) {
impl_.reset(new MessageImpl(buffer, offset));
}
Status Message::Open(
const std::shared_ptr<Buffer>& buffer, std::shared_ptr<Message>* out) {
std::shared_ptr<Message> result(new Message());
Status Message::Open(const std::shared_ptr<Buffer>& buffer, int64_t offset,
std::shared_ptr<Message>* out) {
// ctor is private
const flatbuf::Message* message = flatbuf::GetMessage(buffer->data());
// TODO(wesm): verify message
result->impl_.reset(new MessageImpl(buffer, message));
*out = result;
return Status::OK();
*out = std::shared_ptr<Message>(new Message(buffer, offset));
return (*out)->impl_->Open();
}
Message::Type Message::type() const {
@@ -101,20 +105,12 @@ int64_t Message::body_length() const {
return impl_->body_length();
}
std::shared_ptr<Message> Message::get_shared_ptr() {
return this->shared_from_this();
}
std::shared_ptr<SchemaMessage> Message::GetSchema() {
return std::make_shared<SchemaMessage>(this->shared_from_this(), impl_->header());
}
// ----------------------------------------------------------------------
// SchemaMessage
// SchemaMetadata
class SchemaMessage::SchemaMessageImpl {
class SchemaMetadata::SchemaMetadataImpl {
public:
explicit SchemaMessageImpl(const void* schema)
explicit SchemaMetadataImpl(const void* schema)
: schema_(static_cast<const flatbuf::Schema*>(schema)) {}
const flatbuf::Field* field(int i) const { return schema_->fields()->Get(i); }
@@ -125,22 +121,29 @@ class SchemaMessage::SchemaMessageImpl {
const flatbuf::Schema* schema_;
};
SchemaMessage::SchemaMessage(
const std::shared_ptr<Message>& message, const void* schema) {
SchemaMetadata::SchemaMetadata(
const std::shared_ptr<Message>& message, const void* flatbuf) {
message_ = message;
impl_.reset(new SchemaMessageImpl(schema));
impl_.reset(new SchemaMetadataImpl(flatbuf));
}
int SchemaMessage::num_fields() const {
SchemaMetadata::SchemaMetadata(const std::shared_ptr<Message>& message) {
message_ = message;
impl_.reset(new SchemaMetadataImpl(message->impl_->header()));
}
SchemaMetadata::~SchemaMetadata() {}
int SchemaMetadata::num_fields() const {
return impl_->num_fields();
}
Status SchemaMessage::GetField(int i, std::shared_ptr<Field>* out) const {
Status SchemaMetadata::GetField(int i, std::shared_ptr<Field>* out) const {
const flatbuf::Field* field = impl_->field(i);
return FieldFromFlatbuffer(field, out);
}
Status SchemaMessage::GetSchema(std::shared_ptr<Schema>* out) const {
Status SchemaMetadata::GetSchema(std::shared_ptr<Schema>* out) const {
std::vector<std::shared_ptr<Field>> fields(num_fields());
for (int i = 0; i < this->num_fields(); ++i) {
RETURN_NOT_OK(GetField(i, &fields[i]));
@@ -150,11 +153,11 @@ Status SchemaMessage::GetSchema(std::shared_ptr<Schema>* out) const {
}
// ----------------------------------------------------------------------
// RecordBatchMessage
// RecordBatchMetadata
class RecordBatchMessage::RecordBatchMessageImpl {
class RecordBatchMetadata::RecordBatchMetadataImpl {
public:
explicit RecordBatchMessageImpl(const void* batch)
explicit RecordBatchMetadataImpl(const void* batch)
: batch_(static_cast<const flatbuf::RecordBatch*>(batch)) {
nodes_ = batch_->nodes();
buffers_ = batch_->buffers();
@@ -176,19 +179,29 @@ class RecordBatchMessage::RecordBatchMessageImpl {
const flatbuffers::Vector<const flatbuf::Buffer*>* buffers_;
};
std::shared_ptr<RecordBatchMessage> Message::GetRecordBatch() {
return std::make_shared<RecordBatchMessage>(this->shared_from_this(), impl_->header());
RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Message>& message) {
message_ = message;
impl_.reset(new RecordBatchMetadataImpl(message->impl_->header()));
}
RecordBatchMessage::RecordBatchMessage(
const std::shared_ptr<Message>& message, const void* batch) {
message_ = message;
impl_.reset(new RecordBatchMessageImpl(batch));
RecordBatchMetadata::RecordBatchMetadata(
const std::shared_ptr<Buffer>& buffer, int64_t offset) {
message_ = nullptr;
buffer_ = buffer;
const flatbuf::RecordBatch* metadata =
flatbuffers::GetRoot<flatbuf::RecordBatch>(buffer->data() + offset);
// TODO(wesm): validate table
impl_.reset(new RecordBatchMetadataImpl(metadata));
}
RecordBatchMetadata::~RecordBatchMetadata() {}
// TODO(wesm): Copying the flatbuffer data isn't great, but this will do for
// now
FieldMetadata RecordBatchMessage::field(int i) const {
FieldMetadata RecordBatchMetadata::field(int i) const {
const flatbuf::FieldNode* node = impl_->field(i);
FieldMetadata result;
@@ -197,7 +210,7 @@ FieldMetadata RecordBatchMessage::field(int i) const {
return result;
}
BufferMetadata RecordBatchMessage::buffer(int i) const {
BufferMetadata RecordBatchMetadata::buffer(int i) const {
const flatbuf::Buffer* buffer = impl_->buffer(i);
BufferMetadata result;
@@ -207,15 +220,15 @@ BufferMetadata RecordBatchMessage::buffer(int i) const {
return result;
}
int32_t RecordBatchMessage::length() const {
int32_t RecordBatchMetadata::length() const {
return impl_->length();
}
int RecordBatchMessage::num_buffers() const {
int RecordBatchMetadata::num_buffers() const {
return impl_->num_buffers();
}
int RecordBatchMessage::num_fields() const {
int RecordBatchMetadata::num_fields() const {
return impl_->num_fields();
}
@@ -268,11 +281,13 @@ class FileFooter::FileFooterImpl {
MetadataVersion::type version() const {
switch (footer_->version()) {
case flatbuf::MetadataVersion_V1_SNAPSHOT:
return MetadataVersion::V1_SNAPSHOT;
case flatbuf::MetadataVersion_V1:
return MetadataVersion::V1;
case flatbuf::MetadataVersion_V2:
return MetadataVersion::V2;
// Add cases as other versions become available
default:
return MetadataVersion::V1_SNAPSHOT;
return MetadataVersion::V2;
}
}
@@ -285,7 +300,7 @@ class FileFooter::FileFooterImpl {
}
Status GetSchema(std::shared_ptr<Schema>* out) const {
auto schema_msg = std::make_shared<SchemaMessage>(nullptr, footer_->schema());
auto schema_msg = std::make_shared<SchemaMetadata>(nullptr, footer_->schema());
return schema_msg->GetSchema(out);
}
cpp/src/arrow/ipc/metadata.h
+27 -23
View File
@@ -42,7 +42,7 @@ class OutputStream;
namespace ipc {
struct MetadataVersion {
enum type { V1_SNAPSHOT };
enum type { V1, V2 };
};
//----------------------------------------------------------------------
@@ -58,10 +58,14 @@ Status WriteSchema(const Schema* schema, std::shared_ptr<Buffer>* out);
class Message;
// Container for serialized Schema metadata contained in an IPC message
class ARROW_EXPORT SchemaMessage {
class ARROW_EXPORT SchemaMetadata {
public:
explicit SchemaMetadata(const std::shared_ptr<Message>& message);
// Accepts an opaque flatbuffer pointer
SchemaMessage(const std::shared_ptr<Message>& message, const void* schema);
SchemaMetadata(const std::shared_ptr<Message>& message, const void* schema);
~SchemaMetadata();
int num_fields() const;
@@ -76,8 +80,8 @@ class ARROW_EXPORT SchemaMessage {
// Parent, owns the flatbuffer data
std::shared_ptr<Message> message_;
class SchemaMessageImpl;
std::unique_ptr<SchemaMessageImpl> impl_;
class SchemaMetadataImpl;
std::unique_ptr<SchemaMetadataImpl> impl_;
};
// Field metadata
@@ -93,10 +97,13 @@ struct BufferMetadata {
};
// Container for serialized record batch metadata contained in an IPC message
class ARROW_EXPORT RecordBatchMessage {
class ARROW_EXPORT RecordBatchMetadata {
public:
// Accepts an opaque flatbuffer pointer
RecordBatchMessage(const std::shared_ptr<Message>& message, const void* batch_meta);
explicit RecordBatchMetadata(const std::shared_ptr<Message>& message);
RecordBatchMetadata(const std::shared_ptr<Buffer>& message, int64_t offset);
~RecordBatchMetadata();
FieldMetadata field(int i) const;
BufferMetadata buffer(int i) const;
@@ -108,37 +115,34 @@ class ARROW_EXPORT RecordBatchMessage {
private:
// Parent, owns the flatbuffer data
std::shared_ptr<Message> message_;
std::shared_ptr<Buffer> buffer_;
class RecordBatchMessageImpl;
std::unique_ptr<RecordBatchMessageImpl> impl_;
class RecordBatchMetadataImpl;
std::unique_ptr<RecordBatchMetadataImpl> impl_;
};
class ARROW_EXPORT DictionaryBatchMessage {
class ARROW_EXPORT DictionaryBatchMetadata {
public:
int64_t id() const;
std::unique_ptr<RecordBatchMessage> data() const;
std::unique_ptr<RecordBatchMetadata> data() const;
};
class ARROW_EXPORT Message : public std::enable_shared_from_this<Message> {
class ARROW_EXPORT Message {
public:
enum Type { NONE, SCHEMA, DICTIONARY_BATCH, RECORD_BATCH };
static Status Open(
const std::shared_ptr<Buffer>& buffer, std::shared_ptr<Message>* out);
std::shared_ptr<Message> get_shared_ptr();
static Status Open(const std::shared_ptr<Buffer>& buffer, int64_t offset,
std::shared_ptr<Message>* out);
int64_t body_length() const;
Type type() const;
// These methods only to be invoked if you have checked the message type
std::shared_ptr<SchemaMessage> GetSchema();
std::shared_ptr<RecordBatchMessage> GetRecordBatch();
std::shared_ptr<DictionaryBatchMessage> GetDictionaryBatch();
private:
Message();
Message(const std::shared_ptr<Buffer>& buffer, int64_t offset);
friend class RecordBatchMetadata;
friend class SchemaMetadata;
// Hide serialization details from user API
class MessageImpl;
cpp/src/arrow/ipc/test-common.h
+7 -8
View File
@@ -39,8 +39,7 @@
namespace arrow {
namespace ipc {
const auto kInt32 = std::make_shared<Int32Type>();
const auto kListInt32 = list(kInt32);
const auto kListInt32 = list(int32());
const auto kListListInt32 = list(kListInt32);
Status MakeRandomInt32Array(
@@ -99,8 +98,8 @@ Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
const int length = 1000;
// Make the schema
auto f0 = std::make_shared<Field>("f0", kInt32);
auto f1 = std::make_shared<Field>("f1", kInt32);
auto f0 = std::make_shared<Field>("f0", int32());
auto f1 = std::make_shared<Field>("f1", int32());
std::shared_ptr<Schema> schema(new Schema({f0, f1}));
// Example data
@@ -161,7 +160,7 @@ Status MakeListRecordBatch(std::shared_ptr<RecordBatch>* out) {
// Make the schema
auto f0 = std::make_shared<Field>("f0", kListInt32);
auto f1 = std::make_shared<Field>("f1", kListListInt32);
auto f2 = std::make_shared<Field>("f2", kInt32);
auto f2 = std::make_shared<Field>("f2", int32());
std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
// Example data
@@ -184,7 +183,7 @@ Status MakeZeroLengthRecordBatch(std::shared_ptr<RecordBatch>* out) {
// Make the schema
auto f0 = std::make_shared<Field>("f0", kListInt32);
auto f1 = std::make_shared<Field>("f1", kListListInt32);
auto f2 = std::make_shared<Field>("f2", kInt32);
auto f2 = std::make_shared<Field>("f2", int32());
std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
// Example data
@@ -205,7 +204,7 @@ Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
// Make the schema
auto f0 = std::make_shared<Field>("f0", kListInt32);
auto f1 = std::make_shared<Field>("f1", kListListInt32);
auto f2 = std::make_shared<Field>("f2", kInt32);
auto f2 = std::make_shared<Field>("f2", int32());
std::shared_ptr<Schema> schema(new Schema({f0, f1, f2}));
// Example data
@@ -226,7 +225,7 @@ Status MakeNonNullRecordBatch(std::shared_ptr<RecordBatch>* out) {
Status MakeDeeplyNestedList(std::shared_ptr<RecordBatch>* out) {
const int batch_length = 5;
TypePtr type = kInt32;
TypePtr type = int32();
MemoryPool* pool = default_memory_pool();
ArrayPtr array;
cpp/src/arrow/ipc/util.h
+2 -4
View File
@@ -28,12 +28,10 @@ namespace arrow {
namespace ipc {
// Align on 8-byte boundaries
static constexpr int kArrowAlignment = 8;
// Buffers are padded to 64-byte boundaries (for SIMD)
static constexpr int kArrowBufferAlignment = 64;
static constexpr int kArrowAlignment = 64;
static constexpr uint8_t kPaddingBytes[kArrowBufferAlignment] = {0};
static constexpr uint8_t kPaddingBytes[kArrowAlignment] = {0};
static inline int64_t PaddedLength(int64_t nbytes, int64_t alignment = kArrowAlignment) {
return ((nbytes + alignment - 1) / alignment) * alignment;
cpp/src/arrow/pretty_print-test.cc
+87
View File
@@ -0,0 +1,87 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <memory>
#include <sstream>
#include <vector>
#include "gtest/gtest.h"
#include "arrow/array.h"
#include "arrow/pretty_print.h"
#include "arrow/test-util.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
#include "arrow/types/list.h"
#include "arrow/types/primitive.h"
#include "arrow/types/string.h"
#include "arrow/types/struct.h"
namespace arrow {
class TestArrayPrinter : public ::testing::Test {
public:
void SetUp() {}
void Print(const Array& array) {}
private:
std::ostringstream sink_;
};
template <typename TYPE, typename C_TYPE>
void CheckPrimitive(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
const char* expected) {
std::ostringstream sink;
MemoryPool* pool = default_memory_pool();
typename TypeTraits<TYPE>::BuilderType builder(pool, std::make_shared<TYPE>());
for (size_t i = 0; i < values.size(); ++i) {
if (is_valid[i]) {
ASSERT_OK(builder.Append(values[i]));
} else {
ASSERT_OK(builder.AppendNull());
}
}
std::shared_ptr<Array> array;
ASSERT_OK(builder.Finish(&array));
ASSERT_OK(PrettyPrint(*array.get(), &sink));
std::string result = sink.str();
ASSERT_EQ(std::string(expected, strlen(expected)), result);
}
TEST_F(TestArrayPrinter, PrimitiveType) {
std::vector<bool> is_valid = {true, true, false, true, false};
std::vector<int32_t> values = {0, 1, 2, 3, 4};
static const char* expected = R"expected([0, 1, null, 3, null])expected";
CheckPrimitive<Int32Type, int32_t>(is_valid, values, expected);
std::vector<std::string> values2 = {"foo", "bar", "", "baz", ""};
static const char* ex2 = R"expected(["foo", "bar", null, "baz", null])expected";
CheckPrimitive<StringType, std::string>(is_valid, values2, ex2);
}
} // namespace arrow
cpp/src/arrow/pretty_print.cc
+192
View File
@@ -0,0 +1,192 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <ostream>
#include <string>
#include "arrow/array.h"
#include "arrow/pretty_print.h"
#include "arrow/table.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
#include "arrow/types/list.h"
#include "arrow/types/string.h"
#include "arrow/types/struct.h"
#include "arrow/util/status.h"
namespace arrow {
class ArrayPrinter : public ArrayVisitor {
public:
ArrayPrinter(const Array& array, std::ostream* sink) : array_(array), sink_(sink) {}
Status Print() { return VisitArray(array_); }
Status VisitArray(const Array& array) { return array.Accept(this); }
template <typename T>
typename std::enable_if<IsNumeric<T>::value, void>::type WriteDataValues(
const T& array) {
const auto data = array.raw_data();
for (int i = 0; i < array.length(); ++i) {
if (i > 0) { (*sink_) << ", "; }
if (array.IsNull(i)) {
(*sink_) << "null";
} else {
(*sink_) << data[i];
}
}
}
// String (Utf8), Binary
template <typename T>
typename std::enable_if<std::is_base_of<BinaryArray, T>::value, void>::type
WriteDataValues(const T& array) {
int32_t length;
for (int i = 0; i < array.length(); ++i) {
if (i > 0) { (*sink_) << ", "; }
if (array.IsNull(i)) {
(*sink_) << "null";
} else {
const char* buf = reinterpret_cast<const char*>(array.GetValue(i, &length));
(*sink_) << "\"" << std::string(buf, length) << "\"";
}
}
}
template <typename T>
typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
WriteDataValues(const T& array) {
for (int i = 0; i < array.length(); ++i) {
if (i > 0) { (*sink_) << ", "; }
if (array.IsNull(i)) {
(*sink_) << "null";
} else {
(*sink_) << (array.Value(i) ? "true" : "false");
}
}
}
void OpenArray() { (*sink_) << "["; }
void CloseArray() { (*sink_) << "]"; }
template <typename T>
Status WritePrimitive(const T& array) {
OpenArray();
WriteDataValues(array);
CloseArray();
return Status::OK();
}
template <typename T>
Status WriteVarBytes(const T& array) {
OpenArray();
WriteDataValues(array);
CloseArray();
return Status::OK();
}
Status Visit(const NullArray& array) override { return Status::OK(); }
Status Visit(const BooleanArray& array) override { return WritePrimitive(array); }
Status Visit(const Int8Array& array) override { return WritePrimitive(array); }
Status Visit(const Int16Array& array) override { return WritePrimitive(array); }
Status Visit(const Int32Array& array) override { return WritePrimitive(array); }
Status Visit(const Int64Array& array) override { return WritePrimitive(array); }
Status Visit(const UInt8Array& array) override { return WritePrimitive(array); }
Status Visit(const UInt16Array& array) override { return WritePrimitive(array); }
Status Visit(const UInt32Array& array) override { return WritePrimitive(array); }
Status Visit(const UInt64Array& array) override { return WritePrimitive(array); }
Status Visit(const HalfFloatArray& array) override { return WritePrimitive(array); }
Status Visit(const FloatArray& array) override { return WritePrimitive(array); }
Status Visit(const DoubleArray& array) override { return WritePrimitive(array); }
Status Visit(const StringArray& array) override { return WriteVarBytes(array); }
Status Visit(const BinaryArray& array) override { return WriteVarBytes(array); }
Status Visit(const DateArray& array) override { return Status::NotImplemented("date"); }
Status Visit(const TimeArray& array) override { return Status::NotImplemented("time"); }
Status Visit(const TimestampArray& array) override {
return Status::NotImplemented("timestamp");
}
Status Visit(const IntervalArray& array) override {
return Status::NotImplemented("interval");
}
Status Visit(const DecimalArray& array) override {
return Status::NotImplemented("decimal");
}
Status Visit(const ListArray& array) override {
// auto type = static_cast<const ListType*>(array.type().get());
// for (size_t i = 0; i < fields.size(); ++i) {
// RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
// }
// return WriteChildren(type->children(), {array.values()});
return Status::OK();
}
Status Visit(const StructArray& array) override {
// auto type = static_cast<const StructType*>(array.type().get());
// for (size_t i = 0; i < fields.size(); ++i) {
// RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
// }
// return WriteChildren(type->children(), array.fields());
return Status::OK();
}
Status Visit(const UnionArray& array) override {
return Status::NotImplemented("union");
}
private:
const Array& array_;
std::ostream* sink_;
};
Status PrettyPrint(const Array& arr, std::ostream* sink) {
ArrayPrinter printer(arr, sink);
return printer.Print();
}
Status PrettyPrint(const RecordBatch& batch, std::ostream* sink) {
for (int i = 0; i < batch.num_columns(); ++i) {
const std::string& name = batch.column_name(i);
(*sink) << name << ": ";
RETURN_NOT_OK(PrettyPrint(*batch.column(i).get(), sink));
(*sink) << "\n";
}
return Status::OK();
}
} // namespace arrow
cpp/src/arrow/pretty_print.h
+35
View File
@@ -0,0 +1,35 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#ifndef ARROW_PRETTY_PRINT_H
#define ARROW_PRETTY_PRINT_H
#include <ostream>
#include "arrow/type_fwd.h"
#include "arrow/util/visibility.h"
namespace arrow {
class Status;
Status ARROW_EXPORT PrettyPrint(const RecordBatch& batch, std::ostream* sink);
Status ARROW_EXPORT PrettyPrint(const Array& arr, std::ostream* sink);
} // namespace arrow
#endif // ARROW_PRETTY_PRINT_H
cpp/src/arrow/test-util.h
+30
View File
@@ -32,6 +32,7 @@
#include "arrow/schema.h"
#include "arrow/table.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/buffer.h"
#include "arrow/util/logging.h"
@@ -59,6 +60,14 @@
EXPECT_TRUE(s.ok()); \
} while (0)
// Alias MSVC popcount to GCC name
#ifdef _MSC_VER
#include <intrin.h>
#define __builtin_popcount __popcnt
#include <nmmintrin.h>
#define __builtin_popcountll _mm_popcnt_u64
#endif
namespace arrow {
class TestBase : public ::testing::Test {
@@ -242,6 +251,27 @@ Status MakeRandomBytePoolBuffer(int32_t length, MemoryPool* pool,
}
} // namespace test
template <typename TYPE, typename C_TYPE>
void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& is_valid,
const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
std::shared_ptr<Buffer> values_buffer;
std::shared_ptr<Buffer> values_bitmap;
ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer));
ASSERT_OK(test::GetBitmapFromBoolVector(is_valid, &values_bitmap));
using ArrayType = typename TypeTraits<TYPE>::ArrayType;
int32_t null_count = 0;
for (bool val : is_valid) {
if (!val) { ++null_count; }
}
*out = std::make_shared<ArrayType>(type, static_cast<int32_t>(values.size()),
values_buffer, null_count, values_bitmap);
}
} // namespace arrow
#endif // ARROW_TEST_UTIL_H_
cpp/src/arrow/type.cc
+42 -4
View File
@@ -105,10 +105,6 @@ std::string UnionType::ToString() const {
return s.str();
}
int NullType::bit_width() const {
return 0;
}
std::string NullType::ToString() const {
return name();
}
@@ -187,4 +183,46 @@ std::shared_ptr<Field> field(
return std::make_shared<Field>(name, type, nullable, dictionary);
}
static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1);
static const BufferDescr kOffsetBuffer(BufferType::OFFSET, 32);
static const BufferDescr kTypeBuffer(BufferType::TYPE, 32);
static const BufferDescr kBooleanBuffer(BufferType::DATA, 1);
static const BufferDescr kValues64(BufferType::DATA, 64);
static const BufferDescr kValues32(BufferType::DATA, 32);
static const BufferDescr kValues16(BufferType::DATA, 16);
static const BufferDescr kValues8(BufferType::DATA, 8);
std::vector<BufferDescr> FixedWidthType::GetBufferLayout() const {
return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())};
}
std::vector<BufferDescr> NullType::GetBufferLayout() const {
return {};
}
std::vector<BufferDescr> BinaryType::GetBufferLayout() const {
return {kValidityBuffer, kOffsetBuffer, kValues8};
}
std::vector<BufferDescr> ListType::GetBufferLayout() const {
return {kValidityBuffer, kOffsetBuffer};
}
std::vector<BufferDescr> StructType::GetBufferLayout() const {
return {kValidityBuffer, kTypeBuffer};
}
std::vector<BufferDescr> UnionType::GetBufferLayout() const {
if (mode == UnionMode::SPARSE) {
return {kValidityBuffer, kTypeBuffer};
} else {
return {kValidityBuffer, kTypeBuffer, kOffsetBuffer};
}
}
std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
// TODO(wesm)
return {};
}
} // namespace arrow
cpp/src/arrow/type.h
+56 -17
View File
@@ -101,6 +101,20 @@ struct Type {
};
};
enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY };
class BufferDescr {
public:
BufferDescr(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {}
BufferType type() const { return type_; }
int bit_width() const { return bit_width_; }
private:
BufferType type_;
int bit_width_;
};
struct ARROW_EXPORT DataType {
Type::type type;
@@ -129,12 +143,18 @@ struct ARROW_EXPORT DataType {
virtual Status Accept(TypeVisitor* visitor) const = 0;
virtual std::string ToString() const = 0;
virtual std::vector<BufferDescr> GetBufferLayout() const = 0;
};
typedef std::shared_ptr<DataType> TypePtr;
struct ARROW_EXPORT FixedWidthMeta {
struct ARROW_EXPORT FixedWidthType : public DataType {
using DataType::DataType;
virtual int bit_width() const = 0;
std::vector<BufferDescr> GetBufferLayout() const override;
};
struct ARROW_EXPORT IntegerMeta {
@@ -184,12 +204,12 @@ struct ARROW_EXPORT Field {
};
typedef std::shared_ptr<Field> FieldPtr;
struct PrimitiveCType : public DataType {
using DataType::DataType;
struct ARROW_EXPORT PrimitiveCType : public FixedWidthType {
using FixedWidthType::FixedWidthType;
};
template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
struct ARROW_EXPORT CTypeImpl : public PrimitiveCType, public FixedWidthMeta {
struct ARROW_EXPORT CTypeImpl : public PrimitiveCType {
using c_type = C_TYPE;
static constexpr Type::type type_id = TYPE_ID;
@@ -204,16 +224,17 @@ struct ARROW_EXPORT CTypeImpl : public PrimitiveCType, public FixedWidthMeta {
std::string ToString() const override { return std::string(DERIVED::name()); }
};
struct ARROW_EXPORT NullType : public DataType, public FixedWidthMeta {
struct ARROW_EXPORT NullType : public DataType {
static constexpr Type::type type_id = Type::NA;
NullType() : DataType(Type::NA) {}
int bit_width() const override;
Status Accept(TypeVisitor* visitor) const override;
std::string ToString() const override;
static std::string name() { return "null"; }
std::vector<BufferDescr> GetBufferLayout() const override;
};
template <typename DERIVED, Type::type TYPE_ID, typename C_TYPE>
@@ -221,10 +242,10 @@ struct IntegerTypeImpl : public CTypeImpl<DERIVED, TYPE_ID, C_TYPE>, public Inte
bool is_signed() const override { return std::is_signed<C_TYPE>::value; }
};
struct ARROW_EXPORT BooleanType : public DataType, FixedWidthMeta {
struct ARROW_EXPORT BooleanType : public FixedWidthType {
static constexpr Type::type type_id = Type::BOOL;
BooleanType() : DataType(Type::BOOL) {}
BooleanType() : FixedWidthType(Type::BOOL) {}
Status Accept(TypeVisitor* visitor) const override;
std::string ToString() const override;
@@ -306,6 +327,8 @@ struct ARROW_EXPORT ListType : public DataType, public NoExtraMeta {
std::string ToString() const override;
static std::string name() { return "list"; }
std::vector<BufferDescr> GetBufferLayout() const override;
};
// BinaryType type is reprsents lists of 1-byte values.
@@ -318,6 +341,8 @@ struct ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
std::string ToString() const override;
static std::string name() { return "binary"; }
std::vector<BufferDescr> GetBufferLayout() const override;
protected:
// Allow subclasses to change the logical type.
explicit BinaryType(Type::type logical_type) : DataType(logical_type) {}
@@ -345,6 +370,8 @@ struct ARROW_EXPORT StructType : public DataType, public NoExtraMeta {
Status Accept(TypeVisitor* visitor) const override;
std::string ToString() const override;
static std::string name() { return "struct"; }
std::vector<BufferDescr> GetBufferLayout() const override;
};
struct ARROW_EXPORT DecimalType : public DataType {
@@ -358,6 +385,8 @@ struct ARROW_EXPORT DecimalType : public DataType {
Status Accept(TypeVisitor* visitor) const override;
std::string ToString() const override;
static std::string name() { return "decimal"; }
std::vector<BufferDescr> GetBufferLayout() const override;
};
enum class UnionMode : char { SPARSE, DENSE };
@@ -375,14 +404,20 @@ struct ARROW_EXPORT UnionType : public DataType {
static std::string name() { return "union"; }
Status Accept(TypeVisitor* visitor) const override;
std::vector<BufferDescr> GetBufferLayout() const override;
UnionMode mode;
std::vector<uint8_t> type_ids;
};
struct ARROW_EXPORT DateType : public DataType, public NoExtraMeta {
struct ARROW_EXPORT DateType : public FixedWidthType {
static constexpr Type::type type_id = Type::DATE;
DateType() : DataType(Type::DATE) {}
using c_type = int32_t;
DateType() : FixedWidthType(Type::DATE) {}
int bit_width() const override { return sizeof(c_type) * 8; }
Status Accept(TypeVisitor* visitor) const override;
std::string ToString() const override { return name(); }
@@ -391,13 +426,17 @@ struct ARROW_EXPORT DateType : public DataType, public NoExtraMeta {
enum class TimeUnit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 };
struct ARROW_EXPORT TimeType : public DataType {
struct ARROW_EXPORT TimeType : public FixedWidthType {
static constexpr Type::type type_id = Type::TIME;
using Unit = TimeUnit;
using c_type = int64_t;
TimeUnit unit;
explicit TimeType(TimeUnit unit = TimeUnit::MILLI) : DataType(Type::TIME), unit(unit) {}
int bit_width() const override { return sizeof(c_type) * 8; }
explicit TimeType(TimeUnit unit = TimeUnit::MILLI)
: FixedWidthType(Type::TIME), unit(unit) {}
TimeType(const TimeType& other) : TimeType(other.unit) {}
Status Accept(TypeVisitor* visitor) const override;
@@ -405,7 +444,7 @@ struct ARROW_EXPORT TimeType : public DataType {
static std::string name() { return "time"; }
};
struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
struct ARROW_EXPORT TimestampType : public FixedWidthType {
using Unit = TimeUnit;
typedef int64_t c_type;
@@ -416,7 +455,7 @@ struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
TimeUnit unit;
explicit TimestampType(TimeUnit unit = TimeUnit::MILLI)
: DataType(Type::TIMESTAMP), unit(unit) {}
: FixedWidthType(Type::TIMESTAMP), unit(unit) {}
TimestampType(const TimestampType& other) : TimestampType(other.unit) {}
@@ -425,10 +464,10 @@ struct ARROW_EXPORT TimestampType : public DataType, public FixedWidthMeta {
static std::string name() { return "timestamp"; }
};
struct ARROW_EXPORT IntervalType : public DataType, public FixedWidthMeta {
struct ARROW_EXPORT IntervalType : public FixedWidthType {
enum class Unit : char { YEAR_MONTH = 0, DAY_TIME = 1 };
typedef int64_t c_type;
using c_type = int64_t;
static constexpr Type::type type_id = Type::INTERVAL;
int bit_width() const override { return sizeof(int64_t) * 8; }
@@ -436,7 +475,7 @@ struct ARROW_EXPORT IntervalType : public DataType, public FixedWidthMeta {
Unit unit;
explicit IntervalType(Unit unit = Unit::YEAR_MONTH)
: DataType(Type::INTERVAL), unit(unit) {}
: FixedWidthType(Type::INTERVAL), unit(unit) {}
IntervalType(const IntervalType& other) : IntervalType(other.unit) {}
cpp/src/arrow/type_traits.h
+6
View File
@@ -192,6 +192,12 @@ struct IsFloatingPoint {
static constexpr bool value = std::is_floating_point<c_type>::value;
};
template <typename T>
struct IsNumeric {
PRIMITIVE_TRAITS(T);
static constexpr bool value = std::is_arithmetic<c_type>::value;
};
} // namespace arrow
#endif // ARROW_TYPE_TRAITS_H
cpp/src/arrow/types/primitive.cc
+1 -1
View File
@@ -49,7 +49,7 @@ bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
const uint8_t* this_data = raw_data_;
const uint8_t* other_data = other.raw_data_;
auto size_meta = dynamic_cast<const FixedWidthMeta*>(type_.get());
auto size_meta = dynamic_cast<const FixedWidthType*>(type_.get());
int value_byte_size = size_meta->bit_width() / 8;
DCHECK_GT(value_byte_size, 0);
cpp/src/arrow/util/CMakeLists.txt
+14 -11
View File
@@ -40,17 +40,20 @@ if (ARROW_BUILD_TESTS)
test_main.cc)
if (APPLE)
target_link_libraries(arrow_test_main
gtest
dl)
set_target_properties(arrow_test_main
PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
target_link_libraries(arrow_test_main
gtest
dl)
set_target_properties(arrow_test_main
PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
elseif(MSVC)
target_link_libraries(arrow_test_main
gtest)
else()
target_link_libraries(arrow_test_main
gtest
pthread
dl
)
target_link_libraries(arrow_test_main
gtest
pthread
dl
)
endif()
endif()
@@ -71,4 +74,4 @@ endif()
ADD_ARROW_TEST(bit-util-test)
ADD_ARROW_TEST(buffer-test)
ADD_ARROW_TEST(memory-pool-test)
ADD_ARROW_TEST(status-test)
ADD_ARROW_TEST(status-test)
cpp/src/arrow/util/bit-util.h
+4
View File
@@ -78,6 +78,10 @@ static inline bool IsMultipleOf64(int64_t n) {
return (n & 63) == 0;
}
static inline bool IsMultipleOf8(int64_t n) {
return (n & 7) == 0;
}
inline int64_t RoundUpToMultipleOf64(int64_t num) {
// TODO(wesm): is this definitely needed?
// DCHECK_GE(num, 0);
cpp/src/arrow/util/buffer.h
+2 -2
View File
@@ -103,7 +103,7 @@ class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
// Construct a view on passed buffer at the indicated offset and length. This
// function cannot fail and does not error checking (except in debug builds)
std::shared_ptr<Buffer> SliceBuffer(
ARROW_EXPORT std::shared_ptr<Buffer> SliceBuffer(
const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
// A Buffer whose contents can be mutated. May or may not own its data.
@@ -154,7 +154,7 @@ class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
MemoryPool* pool_;
};
class BufferBuilder {
class ARROW_EXPORT BufferBuilder {
public:
explicit BufferBuilder(MemoryPool* pool)
: pool_(pool), data_(nullptr), capacity_(0), size_(0) {}
cpp/src/arrow/util/memory-pool.cc
+14
View File
@@ -33,6 +33,15 @@ namespace {
Status AllocateAligned(int64_t size, uint8_t** out) {
// TODO(emkornfield) find something compatible with windows
constexpr size_t kAlignment = 64;
#ifdef _MSC_VER
// Special code path for MSVC
*out = reinterpret_cast<uint8_t*>(_aligned_malloc(size, kAlignment));
if (!*out) {
std::stringstream ss;
ss << "malloc of size " << size << " failed";
return Status::OutOfMemory(ss.str());
}
#else
const int result = posix_memalign(reinterpret_cast<void**>(out), kAlignment, size);
if (result == ENOMEM) {
std::stringstream ss;
@@ -45,6 +54,7 @@ Status AllocateAligned(int64_t size, uint8_t** out) {
ss << "invalid alignment parameter: " << kAlignment;
return Status::Invalid(ss.str());
}
#endif
return Status::OK();
}
} // namespace
@@ -83,7 +93,11 @@ int64_t InternalMemoryPool::bytes_allocated() const {
void InternalMemoryPool::Free(uint8_t* buffer, int64_t size) {
std::lock_guard<std::mutex> guard(pool_lock_);
DCHECK_GE(bytes_allocated_, size);
#ifdef _MSC_VER
_aligned_free(buffer);
#else
std::free(buffer);
#endif
bytes_allocated_ -= size;
}
cpp/src/arrow/util/visibility.h
+1
View File
@@ -20,6 +20,7 @@
#if defined(_WIN32) || defined(__CYGWIN__)
#define ARROW_EXPORT __declspec(dllexport)
#define ARROW_NO_EXPORT
#else // Not Windows
#ifndef ARROW_EXPORT
#define ARROW_EXPORT __attribute__((visibility("default")))
dev/release/run-rat.sh
+1 -2
View File
@@ -28,6 +28,7 @@ $RAT $1 \
-e ".*" \
-e mman.h \
-e "*_generated.h" \
-e "*.json" \
-e random.h \
-e status.cc \
-e status.h \
@@ -49,5 +50,3 @@ else
echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
exit 1
fi
format/IPC.md
+106
View File
@@ -15,3 +15,109 @@
# Interprocess messaging / communication (IPC)
## File format
We define a self-contained "file format" containing an Arrow schema along with
one or more record batches defining a dataset. See [format/File.fbs][1] for the
precise details of the file metadata.
In general, the file looks like:
```
<magic number "ARROW1">
<empty padding bytes [to 8 byte boundary]>
<DICTIONARY 0>
...
<DICTIONARY k - 1>
<RECORD BATCH 0>
...
<RECORD BATCH n - 1>
<METADATA org.apache.arrow.flatbuf.Footer>
<metadata_size: int32>
<magic number "ARROW1">
```
See the File.fbs document for details about the Flatbuffers metadata. The
record batches have a particular structure, defined next.
### Record batches
The record batch metadata is written as a flatbuffer (see
[format/Message.fbs][2] -- the RecordBatch message type) prefixed by its size,
followed by each of the memory buffers in the batch written end to end (with
appropriate alignment and padding):
```
<int32: metadata flatbuffer size>
<metadata: org.apache.arrow.flatbuf.RecordBatch>
<padding bytes [to 8-byte boundary]>
<body: buffers end to end>
```
The `RecordBatch` metadata contains a depth-first (pre-order) flattened set of
field metadata and physical memory buffers (some comments from [Message.fbs][2]
have been shortened / removed):
```
table RecordBatch {
length: int;
nodes: [FieldNode];
buffers: [Buffer];
}
struct FieldNode {
/// The number of value slots in the Arrow array at this level of a nested
/// tree
length: int;
/// The number of observed nulls. Fields with null_count == 0 may choose not
/// to write their physical validity bitmap out as a materialized buffer,
/// instead setting the length of the bitmap buffer to 0.
null_count: int;
}
struct Buffer {
/// The shared memory page id where this buffer is located. Currently this is
/// not used
page: int;
/// The relative offset into the shared memory page where the bytes for this
/// buffer starts
offset: long;
/// The absolute length (in bytes) of the memory buffer. The memory is found
/// from offset (inclusive) to offset + length (non-inclusive).
length: long;
}
```
In the context of a file, the `page` is not used, and the `Buffer` offsets use
as a frame of reference the start of the segment where they are written in the
file. So, while in a general IPC setting these offsets may be anyplace in one
or more shared memory regions, in the file format the offsets start from 0.
The location of a record batch and the size of the metadata block as well as
the body of buffers is stored in the file footer:
```
struct Block {
offset: long;
metaDataLength: int;
bodyLength: long;
}
```
Some notes about this
* The `Block` offset indicates the starting byte of the record batch.
* The metadata length includes the flatbuffer size, the record batch metadata
flatbuffer, and any padding bytes
### Dictionary batches
Dictionary batches have not yet been implemented, while they are provided for
in the metadata. For the time being, the `DICTIONARY` segments shown above in
the file do not appear in any of the file implementations.
[1]: https://github.com/apache/arrow/blob/master/format/File.fbs
[1]: https://github.com/apache/arrow/blob/master/format/Message.fbs
format/Message.fbs
+2 -1
View File
@@ -18,7 +18,8 @@
namespace org.apache.arrow.flatbuf;
enum MetadataVersion:short {
V1_SNAPSHOT
V1,
V2
}
/// ----------------------------------------------------------------------
integration/data/simple.json
+66
View File
@@ -0,0 +1,66 @@
{
"schema": {
"fields": [
{
"name": "foo",
"type": {"name": "int", "isSigned": true, "bitWidth": 32},
"nullable": true, "children": [],
"typeLayout": {
"vectors": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "DATA", "typeBitWidth": 32}
]
}
},
{
"name": "bar",
"type": {"name": "floatingpoint", "precision": "DOUBLE"},
"nullable": true, "children": [],
"typeLayout": {
"vectors": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "DATA", "typeBitWidth": 64}
]
}
},
{
"name": "baz",
"type": {"name": "utf8"},
"nullable": true, "children": [],
"typeLayout": {
"vectors": [
{"type": "VALIDITY", "typeBitWidth": 1},
{"type": "OFFSET", "typeBitWidth": 32},
{"type": "DATA", "typeBitWidth": 8}
]
}
}
]
},
"batches": [
{
"count": 5,
"columns": [
{
"name": "foo",
"count": 5,
"VALIDITY": [1, 0, 1, 1, 1],
"DATA": [1, 2, 3, 4, 5]
},
{
"name": "bar",
"count": 5,
"VALIDITY": [1, 0, 0, 1, 1],
"DATA": [1.0, 2.0, 3.0, 4.0, 5.0]
},
{
"name": "baz",
"count": 5,
"VALIDITY": [1, 0, 0, 1, 1],
"OFFSET": [0, 2, 2, 2, 5, 9],
"DATA": ["aa", "", "", "bbb", "cccc"]
}
]
}
]
}
integration/integration_test.py
+177
View File
@@ -0,0 +1,177 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import argparse
import glob
import itertools
import os
import six
import subprocess
import tempfile
import uuid
ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
def guid():
return uuid.uuid4().hex
def run_cmd(cmd):
if isinstance(cmd, six.string_types):
cmd = cmd.split(' ')
try:
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
# this avoids hiding the stdout / stderr of failed processes
print('Command failed: %s' % ' '.join(cmd))
print('With output:')
print('--------------')
print(e.output)
print('--------------')
raise e
if isinstance(output, six.binary_type):
output = output.decode('utf-8')
return output
class IntegrationRunner(object):
def __init__(self, json_files, testers, debug=False):
self.json_files = json_files
self.testers = testers
self.temp_dir = tempfile.mkdtemp()
self.debug = debug
def run(self):
for producer, consumer in itertools.product(self.testers,
self.testers):
if producer is consumer:
continue
print('-- {0} producing, {1} consuming'.format(producer.name,
consumer.name))
for json_path in self.json_files:
print('Testing with {0}'.format(json_path))
arrow_path = os.path.join(self.temp_dir, guid())
producer.json_to_arrow(json_path, arrow_path)
consumer.validate(json_path, arrow_path)
class Tester(object):
def __init__(self, debug=False):
self.debug = debug
def json_to_arrow(self, json_path, arrow_path):
raise NotImplementedError
def validate(self, json_path, arrow_path):
raise NotImplementedError
class JavaTester(Tester):
ARROW_TOOLS_JAR = os.path.join(ARROW_HOME,
'java/tools/target/arrow-tools-0.1.1-'
'SNAPSHOT-jar-with-dependencies.jar')
name = 'Java'
def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
'org.apache.arrow.tools.Integration']
if arrow_path is not None:
cmd.extend(['-a', arrow_path])
if json_path is not None:
cmd.extend(['-j', json_path])
cmd.extend(['-c', command])
if self.debug:
print(' '.join(cmd))
return run_cmd(cmd)
def validate(self, json_path, arrow_path):
return self._run(arrow_path, json_path, 'VALIDATE')
def json_to_arrow(self, json_path, arrow_path):
return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
class CPPTester(Tester):
CPP_INTEGRATION_EXE = os.environ.get(
'ARROW_CPP_TESTER',
os.path.join(ARROW_HOME,
'cpp/test-build/debug/json-integration-test'))
name = 'C++'
def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
cmd = [self.CPP_INTEGRATION_EXE, '--integration']
if arrow_path is not None:
cmd.append('--arrow=' + arrow_path)
if json_path is not None:
cmd.append('--json=' + json_path)
cmd.append('--mode=' + command)
if self.debug:
print(' '.join(cmd))
return run_cmd(cmd)
def validate(self, json_path, arrow_path):
return self._run(arrow_path, json_path, 'VALIDATE')
def json_to_arrow(self, json_path, arrow_path):
return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
def get_json_files():
glob_pattern = os.path.join(ARROW_HOME, 'integration', 'data', '*.json')
return glob.glob(glob_pattern)
def run_all_tests(debug=False):
testers = [JavaTester(debug=debug), CPPTester(debug=debug)]
json_files = get_json_files()
runner = IntegrationRunner(json_files, testers, debug=debug)
runner.run()
print('-- All tests passed!')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Arrow integration test CLI')
parser.add_argument('--debug', dest='debug', action='store_true',
default=False,
help='Run executables in debug mode as relevant')
args = parser.parse_args()
run_all_tests(debug=args.debug)
java/pom.xml
+3 -3
View File
@@ -24,7 +24,7 @@
<packaging>pom</packaging>
<name>Apache Arrow Java Root POM</name>
<description>Apache arrow is an open source, low latency SQL query engine for Hadoop and NoSQL.</description>
<description>Apache Arrow is open source, in-memory columnar data structures and low-overhead messaging</description>
<url>http://arrow.apache.org/</url>
<properties>
@@ -442,8 +442,8 @@
<scope>test</scope>
</dependency>
<dependency>
<!-- Mockito needs to be on the class path after JUnit (or Hamcrest) as
long as Mockito _contains_ older Hamcrest classes. See arrow-2130. -->
<!-- Mockito needs to be on the class path after JUnit (or Hamcrest) as
long as Mockito _contains_ older Hamcrest classes. See arrow-2130. -->
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>1.9.5</version>
java/tools/pom.xml
+6
View File
@@ -45,6 +45,12 @@
<artifactId>commons-cli</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.0.13</version>
<scope>run</scope>
</dependency>
</dependencies>
<build>
java/tools/src/main/java/org/apache/arrow/tools/Integration.java
+2 -1
View File
@@ -80,7 +80,7 @@ public class Integration {
Schema schema = footer.getSchema();
LOGGER.debug("Input file size: " + arrowFile.length());
LOGGER.debug("Found schema: " + schema);
try (JsonFileWriter writer = new JsonFileWriter(jsonFile);) {
try (JsonFileWriter writer = new JsonFileWriter(jsonFile, JsonFileWriter.config().pretty(true));) {
writer.start(schema);
List<ArrowBlock> recordBatches = footer.getRecordBatches();
for (ArrowBlock rbBlock : recordBatches) {
@@ -220,6 +220,7 @@ public class Integration {
private static void fatalError(String message, Throwable e) {
System.err.println(message);
System.err.println(e.getMessage());
LOGGER.error(message, e);
System.exit(1);
}
java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
+53 -1
View File
@@ -24,9 +24,12 @@ import static org.apache.arrow.tools.ArrowFileTestFixtures.writeData;
import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput;
import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.StringReader;
import java.util.Map;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
@@ -44,6 +47,11 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
public class TestIntegration {
@Rule
@@ -69,7 +77,7 @@ public class TestIntegration {
File testOutFile = testFolder.newFile("testOut.arrow");
testOutFile.delete();
// generate an arow file
// generate an arrow file
writeInput(testInFile, allocator);
Integration integration = new Integration();
@@ -90,6 +98,50 @@ public class TestIntegration {
integration.run(args3);
}
@Test
public void testJSONRoundTripWithVariableWidth() throws Exception {
File testJSONFile = new File("../../integration/data/simple.json");
File testOutFile = testFolder.newFile("testOut.arrow");
File testRoundTripJSONFile = testFolder.newFile("testOut.json");
testOutFile.delete();
testRoundTripJSONFile.delete();
Integration integration = new Integration();
// convert to arrow
String[] args1 = { "-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()};
integration.run(args1);
// convert back to json
String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json", testRoundTripJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()};
integration.run(args2);
BufferedReader orig = readNormalized(testJSONFile);
BufferedReader rt = readNormalized(testRoundTripJSONFile);
String i, o;
int j = 0;
while ((i = orig.readLine()) != null && (o = rt.readLine()) != null) {
Assert.assertEquals("line: " + j, i, o);
++j;
}
}
private ObjectMapper om = new ObjectMapper();
{
DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter();
prettyPrinter.indentArraysWith(NopIndenter.instance);
om.setDefaultPrettyPrinter(prettyPrinter);
om.enable(SerializationFeature.INDENT_OUTPUT);
om.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
}
private BufferedReader readNormalized(File f) throws IOException {
Map<?,?> tree = om.readValue(f, Map.class);
String normalized = om.writeValueAsString(tree);
return new BufferedReader(new StringReader(normalized));
}
@Test
public void testInvalid() throws Exception {
File testValidInFile = testFolder.newFile("testValidIn.arrow");
java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+3 -1
View File
@@ -81,7 +81,9 @@ public class VectorLoader {
try {
vector.loadFieldBuffers(fieldNode, ownBuffers);
} catch (RuntimeException e) {
throw new IllegalArgumentException("Could not load buffers for field " + field, e);
e.printStackTrace();
throw new IllegalArgumentException("Could not load buffers for field " +
field + " error message" + e.getMessage(), e);
}
List<Field> children = field.getChildren();
if (children.size() > 0) {
java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java
+6 -11
View File
@@ -17,6 +17,8 @@
*/
package org.apache.arrow.vector.file;
import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector;
import java.util.ArrayList;
import java.util.List;
@@ -52,10 +54,10 @@ public class ArrowFooter implements FBSerializable {
private static List<ArrowBlock> recordBatches(Footer footer) {
List<ArrowBlock> recordBatches = new ArrayList<>();
Block tempBLock = new Block();
Block tempBlock = new Block();
int recordBatchesLength = footer.recordBatchesLength();
for (int i = 0; i < recordBatchesLength; i++) {
Block block = footer.recordBatches(tempBLock, i);
Block block = footer.recordBatches(tempBlock, i);
recordBatches.add(new ArrowBlock(block.offset(), block.metaDataLength(), block.bodyLength()));
}
return recordBatches;
@@ -88,9 +90,9 @@ public class ArrowFooter implements FBSerializable {
public int writeTo(FlatBufferBuilder builder) {
int schemaIndex = schema.getSchema(builder);
Footer.startDictionariesVector(builder, dictionaries.size());
int dicsOffset = endVector(builder, dictionaries);
int dicsOffset = writeAllStructsToVector(builder, dictionaries);
Footer.startRecordBatchesVector(builder, recordBatches.size());
int rbsOffset = endVector(builder, recordBatches);
int rbsOffset = writeAllStructsToVector(builder, recordBatches);
Footer.startFooter(builder);
Footer.addSchema(builder, schemaIndex);
Footer.addDictionaries(builder, dicsOffset);
@@ -98,13 +100,6 @@ public class ArrowFooter implements FBSerializable {
return Footer.endFooter(builder);
}
private int endVector(FlatBufferBuilder builder, List<ArrowBlock> blocks) {
for (ArrowBlock block : blocks) {
block.writeTo(builder);
}
return builder.endVector();
}
@Override
public int hashCode() {
final int prime = 31;
java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+5 -1
View File
@@ -123,7 +123,11 @@ public class ArrowReader implements AutoCloseable {
if (n != l) {
throw new IllegalStateException(n + " != " + l);
}
RecordBatch recordBatchFB = RecordBatch.getRootAsRecordBatch(buffer.nioBuffer().asReadOnlyBuffer());
// Record batch flatbuffer is prefixed by its size as int32le
final ArrowBuf metadata = buffer.slice(4, recordBatchBlock.getMetadataLength() - 4);
RecordBatch recordBatchFB = RecordBatch.getRootAsRecordBatch(metadata.nioBuffer().asReadOnlyBuffer());
int nodesLength = recordBatchFB.nodesLength();
final ArrowBuf body = buffer.slice(recordBatchBlock.getMetadataLength(), (int)recordBatchBlock.getBodyLength());
List<ArrowFieldNode> nodes = new ArrayList<>();
java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+17 -6
View File
@@ -99,9 +99,10 @@ public class ArrowWriter implements AutoCloseable {
public void writeRecordBatch(ArrowRecordBatch recordBatch) throws IOException {
checkStarted();
align();
// write metadata header
// write metadata header with int32 size prefix
long offset = currentPosition;
write(recordBatch);
write(recordBatch, true);
align();
// write body
long bodyOffset = currentPosition;
@@ -117,6 +118,7 @@ public class ArrowWriter implements AutoCloseable {
if (startPosition != currentPosition) {
writeZeros((int)(startPosition - currentPosition));
}
write(buffer);
if (currentPosition != startPosition + layout.getSize()) {
throw new IllegalStateException("wrong buffer size: " + currentPosition + " != " + startPosition + layout.getSize());
@@ -133,7 +135,9 @@ public class ArrowWriter implements AutoCloseable {
}
private void write(ArrowBuf buffer) throws IOException {
write(buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes()));
ByteBuffer nioBuffer = buffer.nioBuffer(buffer.readerIndex(), buffer.readableBytes());
LOGGER.debug("Writing buffer with size: " + nioBuffer.remaining());
write(nioBuffer);
}
private void checkStarted() throws IOException {
@@ -166,14 +170,21 @@ public class ArrowWriter implements AutoCloseable {
private void writeFooter() throws IOException {
// TODO: dictionaries
write(new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), recordBatches));
write(new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), recordBatches), false);
}
private long write(FBSerializable writer) throws IOException {
private long write(FBSerializable writer, boolean withSizePrefix) throws IOException {
FlatBufferBuilder builder = new FlatBufferBuilder();
int root = writer.writeTo(builder);
builder.finish(root);
return write(builder.dataBuffer());
ByteBuffer buffer = builder.dataBuffer();
if (withSizePrefix) {
writeIntLittleEndian(buffer.remaining());
}
return write(buffer);
}
}
java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+5 -2
View File
@@ -22,6 +22,7 @@ import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
import static com.fasterxml.jackson.core.JsonToken.START_ARRAY;
import static com.fasterxml.jackson.core.JsonToken.START_OBJECT;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET;
import java.io.File;
import java.io.IOException;
@@ -127,11 +128,13 @@ public class JsonFileReader implements AutoCloseable {
ValueVector valueVector = (ValueVector)innerVector;
valueVector.allocateNew();
Mutator mutator = valueVector.getMutator();
mutator.setValueCount(count);
for (int i = 0; i < count; i++) {
int innerVectorCount = vectorType.equals(OFFSET) ? count + 1 : count;
for (int i = 0; i < innerVectorCount; i++) {
parser.nextToken();
setValueFromParser(valueVector, i);
}
mutator.setValueCount(innerVectorCount);
readToken(END_ARRAY);
}
// if children
java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+1 -1
View File
@@ -114,7 +114,7 @@ public class JsonFileWriter implements AutoCloseable {
BufferBacked innerVector = fieldInnerVectors.get(v);
generator.writeArrayFieldStart(vectorType.getName());
ValueVector valueVector = (ValueVector)innerVector;
for (int i = 0; i < valueCount; i++) {
for (int i = 0; i < valueVector.getAccessor().getValueCount(); i++) {
writeValueToGenerator(valueVector, i);
}
generator.writeEndArray();
java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java
+15
View File
@@ -81,4 +81,19 @@ public class ArrowVectorType {
public String toString() {
return getName();
}
@Override
public int hashCode() {
return type;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof ArrowVectorType) {
ArrowVectorType other = (ArrowVectorType) obj;
return type == other.type;
}
return false;
}
}
java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+17 -8
View File
@@ -161,24 +161,27 @@ public class TestArrowFile extends BaseFileTest {
@Test
public void testWriteReadMultipleRBs() throws IOException {
File file = new File("target/mytest_multiple.arrow");
int count = COUNT;
int[] counts = { 10, 5 };
// write
try (
BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
MapVector parent = new MapVector("parent", originalVectorAllocator, null);
FileOutputStream fileOutputStream = new FileOutputStream(file);) {
writeData(count, parent);
VectorUnloader vectorUnloader = newVectorUnloader(parent.getChild("root"));
Schema schema = vectorUnloader.getSchema();
writeData(counts[0], parent);
VectorUnloader vectorUnloader0 = newVectorUnloader(parent.getChild("root"));
Schema schema = vectorUnloader0.getSchema();
Assert.assertEquals(2, schema.getFields().size());
try (ArrowWriter arrowWriter = new ArrowWriter(fileOutputStream.getChannel(), schema);) {
try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch()) {
try (ArrowRecordBatch recordBatch = vectorUnloader0.getRecordBatch()) {
Assert.assertEquals("RB #0", counts[0], recordBatch.getLength());
arrowWriter.writeRecordBatch(recordBatch);
}
parent.allocateNew();
writeData(count, parent);
try (ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch()) {
writeData(counts[1], parent); // if we write the same data we don't catch that the metadata is stored in the wrong order.
VectorUnloader vectorUnloader1 = newVectorUnloader(parent.getChild("root"));
try (ArrowRecordBatch recordBatch = vectorUnloader1.getRecordBatch()) {
Assert.assertEquals("RB #1", counts[1], recordBatch.getLength());
arrowWriter.writeRecordBatch(recordBatch);
}
}
@@ -195,21 +198,27 @@ public class TestArrowFile extends BaseFileTest {
ArrowFooter footer = arrowReader.readFooter();
Schema schema = footer.getSchema();
LOGGER.debug("reading schema: " + schema);
int i = 0;
try (VectorSchemaRoot root = new VectorSchemaRoot(schema, vectorAllocator);) {
VectorLoader vectorLoader = new VectorLoader(root);
List<ArrowBlock> recordBatches = footer.getRecordBatches();
Assert.assertEquals(2, recordBatches.size());
long previousOffset = 0;
for (ArrowBlock rbBlock : recordBatches) {
Assert.assertTrue(rbBlock.getOffset() + " > " + previousOffset, rbBlock.getOffset() > previousOffset);
previousOffset = rbBlock.getOffset();
Assert.assertEquals(0, rbBlock.getOffset() % 8);
Assert.assertEquals(0, rbBlock.getMetadataLength() % 8);
try (ArrowRecordBatch recordBatch = arrowReader.readRecordBatch(rbBlock)) {
Assert.assertEquals("RB #" + i, counts[i], recordBatch.getLength());
List<ArrowBuffer> buffersLayout = recordBatch.getBuffersLayout();
for (ArrowBuffer arrowBuffer : buffersLayout) {
Assert.assertEquals(0, arrowBuffer.getOffset() % 8);
}
vectorLoader.load(recordBatch);
validateContent(count, root);
validateContent(counts[i], root);
}
++i;
}
}
}
python/.gitignore
-10
View File
@@ -12,16 +12,6 @@ Testing/
# Editor temporary/working/backup files
*flymake*
# Compiled source
*.a
*.dll
*.o
*.py[ocd]
*.so
*.dylib
.build_cache_dir
MANIFEST
# Generated sources
*.c
*.cpp
python/doc/conf.py
+4 -1
View File
@@ -59,9 +59,12 @@ extensions = [
'sphinx.ext.doctest',
'sphinx.ext.mathjax',
'sphinx.ext.viewcode',
'numpydoc'
'sphinx.ext.napoleon'
]
# numpydoc configuration
napoleon_use_rtype = False
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
python/pyarrow/array.pyx
+50 -3
View File
@@ -54,6 +54,41 @@ cdef class Array:
@staticmethod
def from_pandas(obj, mask=None):
"""
Create an array from a pandas.Series
Parameters
----------
obj : pandas.Series or numpy.ndarray
vector holding the data
mask : numpy.ndarray, optional
boolean mask if the object is valid or null
Returns
-------
pyarrow.Array
Examples
--------
>>> import pandas as pd
>>> import pyarrow as pa
>>> pa.Array.from_pandas(pd.Series([1, 2]))
<pyarrow.array.Int64Array object at 0x7f674e4c0e10>
[
1,
2
]
>>> import numpy as np
>>> pa.Array.from_pandas(pd.Series([1, 2]), np.array([0, 1], dtype=bool))
<pyarrow.array.Int64Array object at 0x7f9019e11208>
[
1,
NA
]
"""
return from_pandas_series(obj, mask)
property null_count:
@@ -228,6 +263,14 @@ cdef object box_arrow_array(const shared_ptr[CArray]& sp_array):
def from_pylist(object list_obj, DataType type=None):
"""
Convert Python list to Arrow array
Parameters
----------
list_obj : array_like
Returns
-------
pyarrow.array.Array
"""
cdef:
shared_ptr[CArray] sp_array
@@ -246,15 +289,19 @@ def from_pandas_series(object series, object mask=None, timestamps_to_ms=False):
Parameters
----------
series: pandas.Series or numpy.ndarray
series : pandas.Series or numpy.ndarray
mask: pandas.Series or numpy.ndarray
mask : pandas.Series or numpy.ndarray, optional
array to mask null entries in the series
timestamps_to_ms: bool
timestamps_to_ms : bool, optional
Convert datetime columns to ms resolution. This is needed for
compability with other functionality like Parquet I/O which
only supports milliseconds.
Returns
-------
pyarrow.array.Array
"""
cdef:
shared_ptr[CArray] out
python/pyarrow/compat.py
+2
View File
@@ -90,3 +90,5 @@ else:
integer_types = six.integer_types + (np.integer,)
__all__ = []
python/pyarrow/schema.pyx
+9
View File
@@ -110,6 +110,15 @@ cdef class Schema:
self.schema = schema.get()
self.sp_schema = schema
def equals(self, other):
"""
Test if this schema is equal to the other
"""
cdef Schema _other
_other = other
return self.sp_schema.get().Equals(_other.sp_schema)
@classmethod
def from_fields(cls, fields):
cdef:
python/pyarrow/table.pyx
+262 -71
View File
@@ -37,9 +37,13 @@ from pyarrow.compat import frombytes, tobytes
cimport cpython
cdef class ChunkedArray:
'''
"""
Array backed via one or more memory chunks.
Warning
-------
Do not call this class's constructor directly.
'''
"""
def __cinit__(self):
self.chunked_array = NULL
@@ -60,19 +64,42 @@ cdef class ChunkedArray:
def __len__(self):
return self.length()
property null_count:
@property
def null_count(self):
"""
Number of null entires
def __get__(self):
self._check_nullptr()
return self.chunked_array.null_count()
Returns
-------
int
"""
self._check_nullptr()
return self.chunked_array.null_count()
property num_chunks:
@property
def num_chunks(self):
"""
Number of underlying chunks
def __get__(self):
self._check_nullptr()
return self.chunked_array.num_chunks()
Returns
-------
int
"""
self._check_nullptr()
return self.chunked_array.num_chunks()
def chunk(self, i):
"""
Select a chunk by its index
Parameters
----------
i : int
Returns
-------
pyarrow.array.Array
"""
self._check_nullptr()
return box_arrow_array(self.chunked_array.chunk(i))
@@ -83,9 +110,13 @@ cdef class ChunkedArray:
cdef class Column:
'''
"""
Named vector of elements of equal type.
Warning
-------
Do not call this class's constructor directly.
'''
"""
def __cinit__(self):
self.column = NULL
@@ -96,7 +127,11 @@ cdef class Column:
def to_pandas(self):
"""
Convert the arrow::Column to a pandas Series
Convert the arrow::Column to a pandas.Series
Returns
-------
pandas.Series
"""
cdef:
PyObject* arr
@@ -121,34 +156,64 @@ cdef class Column:
self._check_nullptr()
return self.column.length()
property shape:
@property
def shape(self):
"""
Dimensions of this columns
def __get__(self):
self._check_nullptr()
return (self.length(),)
Returns
-------
(int,)
"""
self._check_nullptr()
return (self.length(),)
property null_count:
@property
def null_count(self):
"""
Number of null entires
def __get__(self):
self._check_nullptr()
return self.column.null_count()
Returns
-------
int
"""
self._check_nullptr()
return self.column.null_count()
property name:
@property
def name(self):
"""
Label of the column
def __get__(self):
return frombytes(self.column.name())
Returns
-------
str
"""
return frombytes(self.column.name())
property type:
@property
def type(self):
"""
Type information for this column
def __get__(self):
return box_data_type(self.column.type())
Returns
-------
pyarrow.schema.DataType
"""
return box_data_type(self.column.type())
property data:
@property
def data(self):
"""
The underlying data
def __get__(self):
cdef ChunkedArray chunked_array = ChunkedArray()
chunked_array.init(self.column.data())
return chunked_array
Returns
-------
pyarrow.table.ChunkedArray
"""
cdef ChunkedArray chunked_array = ChunkedArray()
chunked_array.init(self.column.data())
return chunked_array
cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
@@ -187,6 +252,13 @@ cdef _dataframe_to_arrays(df, name, timestamps_to_ms):
cdef class RecordBatch:
"""
Batch of rows of columns of equal length
Warning
-------
Do not call this class's constructor directly, use one of the ``from_*`` methods instead.
"""
def __cinit__(self):
self.batch = NULL
@@ -204,28 +276,48 @@ cdef class RecordBatch:
self._check_nullptr()
return self.batch.num_rows()
property num_columns:
@property
def num_columns(self):
"""
Number of columns
def __get__(self):
self._check_nullptr()
return self.batch.num_columns()
Returns
-------
int
"""
self._check_nullptr()
return self.batch.num_columns()
property num_rows:
@property
def num_rows(self):
"""
Number of rows
def __get__(self):
return len(self)
Due to the definition of a RecordBatch, all columns have the same number of rows.
property schema:
Returns
-------
int
"""
return len(self)
def __get__(self):
cdef Schema schema
self._check_nullptr()
if self._schema is None:
schema = Schema()
schema.init_schema(self.batch.schema())
self._schema = schema
@property
def schema(self):
"""
Schema of the RecordBatch and its columns
return self._schema
Returns
-------
pyarrow.schema.Schema
"""
cdef Schema schema
self._check_nullptr()
if self._schema is None:
schema = Schema()
schema.init_schema(self.batch.schema())
self._schema = schema
return self._schema
def __getitem__(self, i):
cdef Array arr = Array()
@@ -241,6 +333,10 @@ cdef class RecordBatch:
def to_pandas(self):
"""
Convert the arrow::RecordBatch to a pandas DataFrame
Returns
-------
pandas.DataFrame
"""
cdef:
PyObject* np_arr
@@ -264,12 +360,34 @@ cdef class RecordBatch:
def from_pandas(cls, df):
"""
Convert pandas.DataFrame to an Arrow RecordBatch
Parameters
----------
df: pandas.DataFrame
Returns
-------
pyarrow.table.RecordBatch
"""
names, arrays = _dataframe_to_arrays(df, None, False)
return cls.from_arrays(names, arrays)
@staticmethod
def from_arrays(names, arrays):
"""
Construct a RecordBatch from multiple pyarrow.Arrays
Parameters
----------
names: list of str
Labels for the columns
arrays: list of pyarrow.Array
column-wise data vectors
Returns
-------
pyarrow.table.RecordBatch
"""
cdef:
Array arr
RecordBatch result
@@ -345,11 +463,13 @@ def dataframe_from_batches(batches):
cdef class Table:
'''
"""
A collection of top-level named, equal length Arrow arrays.
Do not call this class's constructor directly.
'''
Warning
-------
Do not call this class's constructor directly, use one of the ``from_*`` methods instead.
"""
def __cinit__(self):
self.table = NULL
@@ -378,6 +498,22 @@ cdef class Table:
Convert datetime columns to ms resolution. This is needed for
compability with other functionality like Parquet I/O which
only supports milliseconds.
Returns
-------
pyarrow.table.Table
Examples
--------
>>> import pandas as pd
>>> import pyarrow as pa
>>> df = pd.DataFrame({
... 'int': [1, 2],
... 'str': ['a', 'b']
... })
>>> pa.table.from_pandas_dataframe(df)
<pyarrow.table.Table object at 0x7f05d1fb1b40>
"""
names, arrays = _dataframe_to_arrays(df, name=name,
timestamps_to_ms=timestamps_to_ms)
@@ -395,8 +531,13 @@ cdef class Table:
Names for the table columns
arrays: list of pyarrow.array.Array
Equal-length arrays that should form the table.
name: str
(optional) name for the Table
name: str, optional
name for the Table
Returns
-------
pyarrow.table.Table
"""
cdef:
Array arr
@@ -430,6 +571,10 @@ cdef class Table:
def to_pandas(self):
"""
Convert the arrow::Table to a pandas DataFrame
Returns
-------
pandas.DataFrame
"""
cdef:
PyObject* arr
@@ -450,18 +595,41 @@ cdef class Table:
return pd.DataFrame(dict(zip(names, data)), columns=names)
property name:
@property
def name(self):
"""
Label of the table
def __get__(self):
self._check_nullptr()
return frombytes(self.table.name())
Returns
-------
str
"""
self._check_nullptr()
return frombytes(self.table.name())
property schema:
@property
def schema(self):
"""
Schema of the table and its columns
def __get__(self):
raise box_schema(self.table.schema())
Returns
-------
pyarrow.schema.Schema
"""
return box_schema(self.table.schema())
def column(self, index):
"""
Select a column by its numeric index.
Parameters
----------
index: int
Returns
-------
pyarrow.table.Column
"""
self._check_nullptr()
cdef Column column = Column()
column.init(self.table.column(index))
@@ -471,28 +639,51 @@ cdef class Table:
return self.column(i)
def itercolumns(self):
"""
Iterator over all columns in their numerical order
"""
for i in range(self.num_columns):
yield self.column(i)
property num_columns:
@property
def num_columns(self):
"""
Number of columns in this table
def __get__(self):
self._check_nullptr()
return self.table.num_columns()
Returns
-------
int
"""
self._check_nullptr()
return self.table.num_columns()
property num_rows:
@property
def num_rows(self):
"""
Number of rows in this table.
def __get__(self):
self._check_nullptr()
return self.table.num_rows()
Due to the definition of a table, all columns have the same number of rows.
Returns
-------
int
"""
self._check_nullptr()
return self.table.num_rows()
def __len__(self):
return self.num_rows
property shape:
@property
def shape(self):
"""
Dimensions of the table: (#rows, #columns)
def __get__(self):
return (self.num_rows, self.num_columns)
Returns
-------
(int, int)
"""
return (self.num_rows, self.num_columns)
python/pyarrow/tests/test_schema.py
+17
View File
@@ -69,3 +69,20 @@ class TestTypes(unittest.TestCase):
foo: int32
bar: string
baz: list<item: int8>"""
def test_schema_equals(self):
fields = [
A.field('foo', A.int32()),
A.field('bar', A.string()),
A.field('baz', A.list_(A.int8()))
]
sch1 = A.schema(fields)
print(dir(sch1))
sch2 = A.schema(fields)
assert sch1.equals(sch2)
del fields[-1]
sch3 = A.schema(fields)
assert not sch1.equals(sch3)