mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-29 00:56:31 +08:00
Code Activity

ARROW-3470: [C++] Fix row-wise example

Browse Source 
- Implement the `ADD_EXAMPLE` cmake function with new ctest label
  `example`, also covered by the `runexample` target. This can be
  toggled via the `ARROW_BUILD_EXAMPLES` option which is ON by default.
- Implement fully working `row-wise-conversion-example.cc` and add it to
  the default build.
- Update documentation to embed (manually) the newly created example.
This commit is contained in:
François Saint-Jacques committed 2018-12-12 12:52:35 -05:00
1 parent aa8bb3cc4b
commit 1eba067d02
11 files changed
+327 -194
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

ci/appveyor-cpp-build.bat
+3
View File
@@ -35,6 +35,7 @@ if "%JOB%" == "Static_Crt_Build" (
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=Debug ^
-DARROW_TEST_LINKAGE=static ^
-DARROW_CXXFLAGS="/MP" ^
@@ -53,6 +54,7 @@ if "%JOB%" == "Static_Crt_Build" (
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=Release ^
-DARROW_TEST_LINKAGE=static ^
-DCMAKE_CXX_FLAGS_RELEASE="/MT %CMAKE_CXX_FLAGS_RELEASE%" ^
@@ -79,6 +81,7 @@ if "%JOB%" == "Build_Debug" (
-DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_BUILD_STATIC=OFF ^
-DARROW_CXXFLAGS="/MP" ^
ci/appveyor-cpp-test-cmake-script.bat
+8
View File
@@ -33,6 +33,7 @@ set FLATBUFFERS_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -51,6 +52,7 @@ set GFLAGS_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -69,6 +71,7 @@ set SNAPPY_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -87,6 +90,7 @@ set ZLIB_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -105,6 +109,7 @@ set BROTLI_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -123,6 +128,7 @@ set LZ4_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -141,6 +147,7 @@ set ZSTD_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -166,6 +173,7 @@ set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. 2>output.txt
ci/cpp-msvc-build-main.bat
+1
View File
@@ -49,6 +49,7 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_BUILD_STATIC=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
-DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^
-DCMAKE_CXX_FLAGS_RELEASE="/MD %CMAKE_CXX_FLAGS_RELEASE%" ^
-DARROW_PARQUET=ON ^
ci/travis_before_script_cpp.sh
+1
View File
@@ -67,6 +67,7 @@ else
$CMAKE_COMMON_FLAGS \
-DARROW_BUILD_BENCHMARKS=ON \
-DARROW_BUILD_TESTS=ON \
-DARROW_BUILD_EXAMPLES=ON \
-DARROW_BUILD_UTILITIES=OFF \
-DARROW_INSTALL_NAME_RPATH=OFF"
fi
cpp/CMakeLists.txt
+15
View File
@@ -122,6 +122,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
"Build the Arrow micro benchmarks, default OFF"
OFF)
option(ARROW_BUILD_EXAMPLES
"Build the Arrow examples, default OFF"
OFF)
set(ARROW_TEST_LINKAGE "shared" CACHE STRING
"Linkage of Arrow libraries with unit tests executables. \
static|shared (default shared)")
@@ -447,6 +451,10 @@ if(NOT ARROW_BUILD_BENCHMARKS)
set(NO_BENCHMARKS 1)
endif()
if(NOT ARROW_BUILD_EXAMPLES)
set(NO_EXAMPLES 1)
endif()
if (NOT ARROW_FUZZING)
set(NO_FUZZING 1)
endif()
@@ -735,12 +743,14 @@ pass ARROW_BUILD_SHARED=on")
endif()
# Use shared linking for unit tests if it's available
set(ARROW_TEST_LINK_LIBS ${ARROW_TEST_SHARED_LINK_LIBS})
set(ARROW_EXAMPLE_LINK_LIBS arrow_shared)
else()
if (NOT ARROW_BUILD_STATIC)
message(FATAL_ERROR "If using static linkage for unit tests, must also \
pass ARROW_BUILD_STATIC=on")
endif()
set(ARROW_TEST_LINK_LIBS ${ARROW_TEST_STATIC_LINK_LIBS})
set(ARROW_EXAMPLE_LINK_LIBS arrow_static)
endif()
if (ARROW_BUILD_BENCHMARKS)
@@ -805,6 +815,11 @@ if(ARROW_GANDIVA)
add_subdirectory(src/gandiva)
endif()
if(ARROW_BUILD_EXAMPLES)
add_custom_target(runexample ctest -L example)
add_subdirectory(examples/arrow)
endif()
include(CMakePackageConfigHelpers)
# Makes the project importable from the build directory
cpp/apidoc/tutorials/row_wise_conversion.md
-194
View File
@@ -1,194 +0,0 @@
<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
Convert a vector of row-wise data into an Arrow table
=====================================================
While we want to use columnar data structures to build efficient operations, we
often receive data in a row-wise fashion from other systems. In the following,
we want give a brief introduction into the classes provided by Apache Arrow by
showing how to transform row-wise data into a columnar table.
The data in this example is stored in the following struct:
```
struct data_row {
int64_t id;
double cost;
std::vector<double> cost_components;
};
std::vector<data_row> rows;
```
The final representation should be an `arrow::Table` which in turn is made up of
an `arrow::Schema` and a list of `arrow::Column`. An `arrow::Column` is again a
named collection of one or more `arrow::Array` instances. As the first step, we
will iterate over the data and build up the arrays incrementally. For this task,
we provide `arrow::ArrayBuilder` classes that help in the construction of the
final `arrow::Array` instances.
For each type, Arrow has a specially typed builder class. For the primitive
values `id` and `cost` we can use the respective `arrow::Int64Builder` and
`arrow::DoubleBuilder`. For the `cost_components` vector, we need to have two
builders, a top-level `arrow::ListBuilder` that builds the array of offsets and
a nested `arrow::DoubleBuilder` that constructs the underlying values array that
is referenced by the offsets in the former array.
```
// The builders are more efficient using
// arrow::jemalloc::MemoryPool::default_pool() as this can increase the size of
// the underlying memory regions in-place. At the moment, arrow::jemalloc is only
// supported on Unix systems, not Windows.
using arrow::DoubleBuilder;
using arrow::Int64Builder;
using arrow::ListBuilder;
arrow::MemoryPool* pool = arrow::default_memory_pool();
Int64Builder id_builder(pool);
DoubleBuilder cost_builder(pool);
std::unique_ptr<DoubleBuilder> components_values_builder(new DoubleBuilder(pool));
ListBuilder components_builder(pool, std::move(components_values_builder));
```
Now we can loop over our existing data and insert it into the builders. The
`Append` calls here may fail (e.g. we cannot allocate enough additional memory).
Thus we need to check their return values. For more information on these values,
check the documentation about `arrow::Status`.
```
for (const data_row& row : rows) {
ARROW_RETURN_NOT_OK(id_builder.Append(row.id));
ARROW_RETURN_NOT_OK(cost_builder.Append(row.cost));
// Indicate the start of a new list row. This will memorise the current
// offset in the values builder.
ARROW_RETURN_NOT_OK(components_builder.Append());
// Store the actual values. The final nullptr argument tells the underyling
// builder that all added values are valid, i.e. non-null.
ARROW_RETURN_NOT_OK(components_values_builder->Append(
row.cost_components.data(), row.cost_components.size(),
nullptr);
}
```
At the end, we finalise the arrays, declare the (type) schema and combine them
into a single `arrow::Table`:
```
std::shared_ptr<arrow::Array> id_array;
ARROW_RETURN_NOT_OK(id_builder.Finish(&id_array));
std::shared_ptr<arrow::Array> cost_array;
ARROW_RETURN_NOT_OK(cost_builder.Finish(&cost_array));
std::shared_ptr<arrow::Array> cost_components_array;
ARROW_RETURN_NOT_OK(components_builder.Finish(&cost_components_array));
std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
arrow::field("id", arrow::int64()),
arrow::field("cost", arrow::float64()),
arrow::field("cost_components", arrow::list(arrow::float64()))
};
auto schema = std::make_shared<arrow::Schema>(schema_vector);
std::shared_ptr<arrow::Table> table = arrow::Table::Make(schema,
{id_array, cost_array, cost_components_array});
```
The final `table` variable is the one we then can pass on to other functions
that can consume Apache Arrow memory structures. This object has ownership of
all referenced data, thus we don't have to care about undefined references once
we leave the scope of the function building the table and its underlying arrays.
<!-- TODO: Add an example with nullable entries -->
Converting an Arrow Table back into row-wise representation
===========================================================
To convert an Arrow table back into the same row-wise representation as in the
above section, we first will check that the table conforms to our expected
schema and then will build up the vector of rows incrementally.
For the check if the table is as expected, we can utilise solely its schema.
```
// This is our input that was passed in from the outside.
std::shared_ptr<arrow::Table> table;
std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
arrow::field("id", arrow::int64()),
arrow::field("cost", arrow::float64()),
arrow::field("cost_components", arrow::list(arrow::float64()))
};
auto expected_schema = std::make_shared<arrow::Schema>(schema_vector);
if (!expected_schema->Equals(*table->schema())) {
// The table doesn't have the expected schema thus we cannot directly
// convert it to our target representation.
// TODO: Implement your custom error handling logic here.
}
```
As we have ensured that the table has the expected structure, we can unpack the
underlying arrays. For the primitive columns `id` and `cost` we can use the high
level functions to get the values whereas for the nested column
`cost_components` we need to access the C-pointer to the data to copy its
contents into the resulting `std::vector<double>`. Here we need to be care to
also add the offset to the pointer. This offset is needed to enable zero-copy
slicing operations. While this could be adjusted automatically for double
arrays, this cannot be done for the accompanying bitmap as often the slicing
border would be inside a byte.
```
// For simplicity, we assume that all arrays consist of a single chunk here.
// In a productive implementation this should either be explicitly check or code
// added that can treat chunked arrays.
auto ids = std::static_pointer_cast<arrow::Int64Array>(
table->column(0)->data()->chunk(0));
auto costs = std::static_pointer_cast<arrow::DoubleArray(
table->column(1)->data()->chunk(0));
auto cost_components = std::static_pointer_cast<arrow::ListArray(
table->column(2)->data()->chunk(0));
auto cost_components_values = std::static_pointer_cast<arrow::DoubleArray>(
cost_components->values());
// To enable zero-copy slices, the native values pointer might need to account
// for this slicing offset. This is not needed for the higher level functions
// like Value(…) that already account for this offset internally.
const double* cost_components_values_ptr = cost_components_values->data()
+ cost_components_values->offset();
```
After we have unpacked the arrays from the table, we can iterate over them in a
row-wise fashion and fill our target, row-wise representation.
```
std::vector<data_row> rows;
for (int64_t i = 0; i < table->num_rows(); i++) {
// Another simplification in this example is that we assume that there are
// no null entries, e.g. each row is fill with valid values.
int64_t id = ids->Value(i);
double cost = costs->Value(i);
const double* first = cost_components_values_ptr + cost_components->value_offset(i);
const double* last = cost_components_values_ptr + cost_components->value_offset(i + 1);
std::vector<double> components_vec(first, last);
rows.push_back({id, cost, components_vec});
}
```
cpp/cmake_modules/BuildUtils.cmake
+60
View File
@@ -480,6 +480,66 @@ function(ADD_TEST_CASE REL_TEST_NAME)
LABELS ${ARG_LABELS})
endfunction()
############################################################
# Examples
############################################################
# Add a new example, with or without an executable that should be built.
# If examples are enabled then they will be run along side unit tests with ctest.
# 'make runexample' to build/run only examples.
#
# REL_EXAMPLE_NAME is the name of the example app. It may be a single component
# (e.g. monotime-example) or contain additional components (e.g.
# net/net_util-example). Either way, the last component must be a globally
# unique name.
# The example will registered as unit test with ctest with a label
# of 'example'.
#
# Arguments after the test name will be passed to set_tests_properties().
#
# \arg PREFIX a string to append to the name of the example executable. For
# example, if you have src/arrow/foo/bar-example.cc, then PREFIX "foo" will
# create test executable foo-bar-example
function(ADD_ARROW_EXAMPLE REL_EXAMPLE_NAME)
set(options)
set(one_value_args)
set(multi_value_args EXTRA_LINK_LIBS DEPENDENCIES PREFIX)
cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
if(ARG_UNPARSED_ARGUMENTS)
message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
endif()
if(NO_EXAMPLES)
return()
endif()
get_filename_component(EXAMPLE_NAME ${REL_EXAMPLE_NAME} NAME_WE)
if(ARG_PREFIX)
set(EXAMPLE_NAME "${ARG_PREFIX}-${EXAMPLE_NAME}")
endif()
if(EXISTS ${CMAKE_SOURCE_DIR}/examples/arrow/${REL_EXAMPLE_NAME}.cc)
# This example has a corresponding .cc file, set it up as an executable.
set(EXAMPLE_PATH "${EXECUTABLE_OUTPUT_PATH}/${EXAMPLE_NAME}")
add_executable(${EXAMPLE_NAME} "${REL_EXAMPLE_NAME}.cc")
target_link_libraries(${EXAMPLE_NAME} ${ARROW_EXAMPLE_LINK_LIBS})
add_dependencies(runexample ${EXAMPLE_NAME})
set(NO_COLOR "--color_print=false")
if (ARG_EXTRA_LINK_LIBS)
target_link_libraries(${EXAMPLE_NAME} ${ARG_EXTRA_LINK_LIBS})
endif()
endif()
if (ARG_DEPENDENCIES)
add_dependencies(${EXAMPLE_NAME} ${ARG_DEPENDENCIES})
endif()
add_test(${EXAMPLE_NAME} ${EXAMPLE_PATH})
set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "example")
endfunction()
############################################################
# Fuzzing
############################################################
cpp/examples/arrow/CMakeLists.txt
+18
View File
@@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ADD_ARROW_EXAMPLE(row-wise-conversion-example)
cpp/examples/arrow/row-wise-conversion-example.cc
+190
View File
@@ -0,0 +1,190 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <cstdint>
#include <iostream>
#include <vector>
#include <arrow/api.h>
using arrow::DoubleBuilder;
using arrow::Int64Builder;
using arrow::ListBuilder;
// While we want to use columnar data structures to build efficient operations, we
// often receive data in a row-wise fashion from other systems. In the following,
// we want give a brief introduction into the classes provided by Apache Arrow by
// showing how to transform row-wise data into a columnar table.
//
// The data in this example is stored in the following struct:
struct data_row {
int64_t id;
double cost;
std::vector<double> cost_components;
};
// Transforming a vector of structs into a columnar Table.
//
// The final representation should be an `arrow::Table` which in turn is made up of
// an `arrow::Schema` and a list of `arrow::Column`. An `arrow::Column` is again a
// named collection of one or more `arrow::Array` instances. As the first step, we
// will iterate over the data and build up the arrays incrementally. For this task,
// we provide `arrow::ArrayBuilder` classes that help in the construction of the
// final `arrow::Array` instances.
//
// For each type, Arrow has a specially typed builder class. For the primitive
// values `id` and `cost` we can use the respective `arrow::Int64Builder` and
// `arrow::DoubleBuilder`. For the `cost_components` vector, we need to have two
// builders, a top-level `arrow::ListBuilder` that builds the array of offsets and
// a nested `arrow::DoubleBuilder` that constructs the underlying values array that
// is referenced by the offsets in the former array.
arrow::Status VectorToColumnarTable(const std::vector<struct data_row>& rows,
std::shared_ptr<arrow::Table>* table) {
// The builders are more efficient using
// arrow::jemalloc::MemoryPool::default_pool() as this can increase the size of
// the underlying memory regions in-place. At the moment, arrow::jemalloc is only
// supported on Unix systems, not Windows.
arrow::MemoryPool* pool = arrow::default_memory_pool();
Int64Builder id_builder(pool);
DoubleBuilder cost_builder(pool);
ListBuilder components_builder(pool, std::make_shared<DoubleBuilder>(pool));
// The following builder is owned by components_builder.
DoubleBuilder& cost_components_builder =
*(static_cast<DoubleBuilder*>(components_builder.value_builder()));
// Now we can loop over our existing data and insert it into the builders. The
// `Append` calls here may fail (e.g. we cannot allocate enough additional memory).
// Thus we need to check their return values. For more information on these values,
// check the documentation about `arrow::Status`.
for (const data_row& row : rows) {
ARROW_RETURN_NOT_OK(id_builder.Append(row.id));
ARROW_RETURN_NOT_OK(cost_builder.Append(row.cost));
// Indicate the start of a new list row. This will memorise the current
// offset in the values builder.
ARROW_RETURN_NOT_OK(components_builder.Append());
// Store the actual values. The final nullptr argument tells the underyling
// builder that all added values are valid, i.e. non-null.
ARROW_RETURN_NOT_OK(cost_components_builder.AppendValues(row.cost_components.data(),
row.cost_components.size()));
}
// At the end, we finalise the arrays, declare the (type) schema and combine them
// into a single `arrow::Table`:
std::shared_ptr<arrow::Array> id_array;
ARROW_RETURN_NOT_OK(id_builder.Finish(&id_array));
std::shared_ptr<arrow::Array> cost_array;
ARROW_RETURN_NOT_OK(cost_builder.Finish(&cost_array));
// No need to invoke cost_components_builder.Finish because it is implied by
// the parent builder's Finish invocation.
std::shared_ptr<arrow::Array> cost_components_array;
ARROW_RETURN_NOT_OK(components_builder.Finish(&cost_components_array));
std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
arrow::field("id", arrow::int64()), arrow::field("cost", arrow::float64()),
arrow::field("cost_components", arrow::list(arrow::float64()))};
auto schema = std::make_shared<arrow::Schema>(schema_vector);
// The final `table` variable is the one we then can pass on to other functions
// that can consume Apache Arrow memory structures. This object has ownership of
// all referenced data, thus we don't have to care about undefined references once
// we leave the scope of the function building the table and its underlying arrays.
*table = arrow::Table::Make(schema, {id_array, cost_array, cost_components_array});
return arrow::Status::OK();
}
arrow::Status ColumnarTableToVector(const std::shared_ptr<arrow::Table>& table,
std::vector<struct data_row>* rows) {
// To convert an Arrow table back into the same row-wise representation as in the
// above section, we first will check that the table conforms to our expected
// schema and then will build up the vector of rows incrementally.
//
// For the check if the table is as expected, we can utilise solely its schema.
std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
arrow::field("id", arrow::int64()), arrow::field("cost", arrow::float64()),
arrow::field("cost_components", arrow::list(arrow::float64()))};
auto expected_schema = std::make_shared<arrow::Schema>(schema_vector);
if (!expected_schema->Equals(*table->schema())) {
// The table doesn't have the expected schema thus we cannot directly
// convert it to our target representation.
return arrow::Status::Invalid("Schemas are not matching!");
}
// As we have ensured that the table has the expected structure, we can unpack the
// underlying arrays. For the primitive columns `id` and `cost` we can use the high
// level functions to get the values whereas for the nested column
// `cost_components` we need to access the C-pointer to the data to copy its
// contents into the resulting `std::vector<double>`. Here we need to be care to
// also add the offset to the pointer. This offset is needed to enable zero-copy
// slicing operations. While this could be adjusted automatically for double
// arrays, this cannot be done for the accompanying bitmap as often the slicing
// border would be inside a byte.
auto ids =
std::static_pointer_cast<arrow::Int64Array>(table->column(0)->data()->chunk(0));
auto costs =
std::static_pointer_cast<arrow::DoubleArray>(table->column(1)->data()->chunk(0));
auto cost_components =
std::static_pointer_cast<arrow::ListArray>(table->column(2)->data()->chunk(0));
auto cost_components_values =
std::static_pointer_cast<arrow::DoubleArray>(cost_components->values());
// To enable zero-copy slices, the native values pointer might need to account
// for this slicing offset. This is not needed for the higher level functions
// like Value(…) that already account for this offset internally.
const double* ccv_ptr = cost_components_values->data()->GetValues<double>(1);
for (int64_t i = 0; i < table->num_rows(); i++) {
// Another simplification in this example is that we assume that there are
// no null entries, e.g. each row is fill with valid values.
int64_t id = ids->Value(i);
double cost = costs->Value(i);
const double* first = ccv_ptr + cost_components->value_offset(i);
const double* last = ccv_ptr + cost_components->value_offset(i + 1);
std::vector<double> components_vec(first, last);
rows->push_back({id, cost, components_vec});
}
return arrow::Status::OK();
}
#define EXIT_ON_FAILURE(expr) \
do { \
arrow::Status status_ = (expr); \
if (!status_.ok()) { \
std::cerr << status_.message() << std::endl; \
return EXIT_FAILURE; \
} \
} while (0);
int main(int argc, char** argv) {
std::vector<data_row> rows = {
{1, 1.0, {1.0}}, {2, 2.0, {1.0, 2.0}}, {3, 3.0, {1.0, 2.0, 3.0}}};
std::shared_ptr<arrow::Table> table;
EXIT_ON_FAILURE(VectorToColumnarTable(rows, &table));
std::vector<data_row> expected_rows;
EXIT_ON_FAILURE(ColumnarTableToVector(table, &expected_rows));
assert(rows.size() == expected_rows.size());
return EXIT_SUCCESS;
}
docs/source/cpp/examples.rst
+30
View File
@@ -0,0 +1,30 @@
.. Licensed to the Apache Software Foundation (ASF) under one
.. or more contributor license agreements. See the NOTICE file
.. distributed with this work for additional information
.. regarding copyright ownership. The ASF licenses this file
.. to you under the Apache License, Version 2.0 (the
.. "License"); you may not use this file except in compliance
.. with the License. You may obtain a copy of the License at
.. http://www.apache.org/licenses/LICENSE-2.0
.. Unless required by applicable law or agreed to in writing,
.. software distributed under the License is distributed on an
.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
.. KIND, either express or implied. See the License for the
.. specific language governing permissions and limitations
.. under the License.
.. default-domain:: cpp
.. highlight:: cpp
Examples
========
Row to columnar conversion
--------------------------
The following example converts an array of structs to a :class:`arrow::Table`
instance, and then converts it back to the original array of structs.
.. literalinclude:: ../../../cpp/examples/arrow/row-wise-conversion-example.cc
docs/source/cpp/index.rst
+1
View File
@@ -22,6 +22,7 @@ C++ Implementation
:maxdepth: 2
getting_started
examples
api
.. TODO add "topics" chapter