mirror of
https://github.com/apache/arrow.git
synced 2026-10-29 00:56:31 +08:00
[Ruby] Import
This commit is contained in:
Kouhei Sutou authored and Wes McKinney committed
1 parent
5c5e95a2d5
commit
cb41c7a89d
91 files changed
+7073
-8
No files matched your search
+10
-6
@@ -151,7 +151,7 @@ matrix:
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
|
||||
script:
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
|
||||
# C++ & glib w/ gcc 4.9
|
||||
# C++ & GLib & Ruby w/ gcc 4.9
|
||||
- compiler: gcc
|
||||
language: cpp
|
||||
os: linux
|
||||
@@ -160,13 +160,15 @@ matrix:
|
||||
- CC="gcc-4.9"
|
||||
- CXX="g++-4.9"
|
||||
before_script:
|
||||
- if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
|
||||
- if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh
|
||||
script:
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
|
||||
# [OS X] C++ & glib w/ XCode 8.3 & homebrew
|
||||
- if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh
|
||||
# [OS X] C++ & GLib & Ruby w/ XCode 8.3 & homebrew
|
||||
- compiler: clang
|
||||
osx_image: xcode8.3
|
||||
os: osx
|
||||
@@ -174,12 +176,14 @@ matrix:
|
||||
addons:
|
||||
rvm: 2.2
|
||||
before_script:
|
||||
- if [ $ARROW_CI_C_GLIB_AFFECTED != "1" ]; then exit; fi
|
||||
- if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_install_osx.sh
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library --homebrew
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_before_script_ruby.sh
|
||||
script:
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
|
||||
- if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi
|
||||
- $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh
|
||||
# Rust
|
||||
- language: rust
|
||||
cache: cargo
|
||||
|
||||
@@ -26,7 +26,7 @@ import subprocess
|
||||
|
||||
perr = functools.partial(print, file=sys.stderr)
|
||||
|
||||
LANGUAGE_TOPICS = ['c_glib', 'cpp', 'java', 'js', 'python', 'rust']
|
||||
LANGUAGE_TOPICS = ['c_glib', 'cpp', 'java', 'js', 'python', 'ruby', 'rust']
|
||||
|
||||
ALL_TOPICS = LANGUAGE_TOPICS + ['integration', 'site']
|
||||
|
||||
@@ -141,7 +141,10 @@ def get_affected_topics(affected_files):
|
||||
elif p in ('java', 'js'):
|
||||
affected[p] = True
|
||||
affected['integration'] = True
|
||||
elif p in ('c_glib', 'integration', 'python', 'site', 'rust'):
|
||||
elif p in ('c_glib'):
|
||||
affected[p] = True
|
||||
affected['ruby'] = True
|
||||
elif p in ('integration', 'python', 'site', 'ruby', 'rust'):
|
||||
affected[p] = True
|
||||
|
||||
return affected
|
||||
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -e
|
||||
|
||||
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
|
||||
|
||||
pushd $ARROW_RUBY_DIR/red-arrow
|
||||
bundle install
|
||||
popd
|
||||
|
||||
# TODO: GPU required
|
||||
# pushd $ARROW_RUBY_DIR/red-arrow-gpu
|
||||
# bundle install
|
||||
# popd
|
||||
@@ -29,6 +29,7 @@ export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
|
||||
export ARROW_JAVA_DIR=${TRAVIS_BUILD_DIR}/java
|
||||
export ARROW_JS_DIR=${TRAVIS_BUILD_DIR}/js
|
||||
export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
|
||||
export ARROW_RUBY_DIR=$TRAVIS_BUILD_DIR/ruby
|
||||
export ARROW_RUST_DIR=${TRAVIS_BUILD_DIR}/rust
|
||||
|
||||
if [ $ARROW_TRAVIS_COVERAGE == "1" ]; then
|
||||
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -e
|
||||
|
||||
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
|
||||
|
||||
arrow_ruby_run_test()
|
||||
{
|
||||
local arrow_c_glib_lib_dir=$1
|
||||
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$arrow_c_glib_lib_dir
|
||||
export GI_TYPELIB_PATH=$arrow_c_glib_lib_dir/girepository-1.0
|
||||
test/run-test.rb
|
||||
}
|
||||
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
|
||||
export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
|
||||
|
||||
pushd $ARROW_RUBY_DIR/red-arrow
|
||||
(arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_AUTOTOOLS/lib)
|
||||
if [ -d $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu ]; then
|
||||
(arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu)
|
||||
# else # TODO: Enable this
|
||||
# (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib)
|
||||
fi
|
||||
popd
|
||||
|
||||
# TODO: GPU required
|
||||
# pushd $ARROW_RUBY_DIR/red-arrow-gpu
|
||||
# (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_AUTOTOOLS/lib)
|
||||
# if [ -d $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu ]; then
|
||||
# (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib/$(arch)-linux-gnu)
|
||||
# # else # TODO: Enable this
|
||||
# # (arrow_ruby_run_test $ARROW_C_GLIB_INSTALL_MESON/lib)
|
||||
# fi
|
||||
# popd
|
||||
@@ -0,0 +1,26 @@
|
||||
<!---
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Apache Arrow Ruby
|
||||
|
||||
There are the official Ruby bindings for Apache Arrow.
|
||||
|
||||
[Red Arrow](https://github.com/apache/arrow/tree/master/ruby/red-arrow) is the base Apache Arrow bindings.
|
||||
|
||||
[Red Arrow GPU](https://github.com/apache/arrow/tree/master/ruby/red-arrow-gpu) is the Apache Arrow bindings of GPU part.
|
||||
@@ -0,0 +1,20 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
/lib/arrow-gpu/version.rb
|
||||
|
||||
/pkg/
|
||||
@@ -0,0 +1,22 @@
|
||||
# -*- ruby -*-
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
source "https://rubygems.org/"
|
||||
|
||||
gemspec
|
||||
@@ -0,0 +1,585 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/plasma/fling.cc and src/plasma/fling.h: Apache 2.0
|
||||
|
||||
Copyright 2013 Sharvil Nanavati
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/plasma/thirdparty/ae: Modified / 3-Clause BSD
|
||||
|
||||
Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
* Neither the name of Redis nor the names of its contributors may be used
|
||||
to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/plasma/thirdparty/dlmalloc.c: CC0
|
||||
|
||||
This is a version (aka dlmalloc) of malloc/free/realloc written by
|
||||
Doug Lea and released to the public domain, as explained at
|
||||
http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
|
||||
comments, complaints, performance data, etc to dl@cs.oswego.edu
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/plasma/thirdparty/xxhash: BSD 2-Clause License
|
||||
|
||||
xxHash - Fast Hash algorithm
|
||||
Copyright (C) 2012-2016, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- xxHash homepage: http://www.xxhash.com
|
||||
- xxHash source repository : https://github.com/Cyan4973/xxHash
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/arrow/util (some portions): Apache 2.0, and 3-clause BSD
|
||||
|
||||
Some portions of this module are derived from code in the Chromium project,
|
||||
copyright (c) Google inc and (c) The Chromium Authors and licensed under the
|
||||
Apache 2.0 License or the under the 3-clause BSD license:
|
||||
|
||||
Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from Daniel Lemire's FrameOfReference project.
|
||||
|
||||
https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp
|
||||
|
||||
Copyright: 2013 Daniel Lemire
|
||||
Home page: http://lemire.me/en/
|
||||
Project page: https://github.com/lemire/FrameOfReference
|
||||
License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the TensorFlow project
|
||||
|
||||
Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the NumPy project.
|
||||
|
||||
https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
|
||||
|
||||
https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c
|
||||
|
||||
Copyright (c) 2005-2017, NumPy Developers.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following
|
||||
disclaimer in the documentation and/or other materials provided
|
||||
with the distribution.
|
||||
|
||||
* Neither the name of the NumPy Developers nor the names of any
|
||||
contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the Boost project
|
||||
|
||||
Boost Software License - Version 1.0 - August 17th, 2003
|
||||
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the mapbox/variant project, BSD 3-clause
|
||||
license
|
||||
|
||||
Copyright (c) MapBox
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
- Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
- Redistributions in binary form must reproduce the above copyright notice, this
|
||||
list of conditions and the following disclaimer in the documentation and/or
|
||||
other materials provided with the distribution.
|
||||
- Neither the name "MapBox" nor the names of its contributors may be
|
||||
used to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the Boost project
|
||||
|
||||
Boost Software License - Version 1.0 - August 17th, 2003
|
||||
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the mapbox/variant project, BSD 3-clause
|
||||
license
|
||||
|
||||
Copyright (c) MapBox
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
- Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
- Redistributions in binary form must reproduce the above copyright notice, this
|
||||
list of conditions and the following disclaimer in the documentation and/or
|
||||
other materials provided with the distribution.
|
||||
- Neither the name "MapBox" nor the names of its contributors may be
|
||||
used to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the FlatBuffers project
|
||||
|
||||
Copyright 2014 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the tslib project
|
||||
|
||||
Copyright 2015 Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the jemalloc project
|
||||
|
||||
https://github.com/jemalloc/jemalloc
|
||||
|
||||
Copyright (C) 2002-2017 Jason Evans <jasone@canonware.com>.
|
||||
All rights reserved.
|
||||
Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
|
||||
Copyright (C) 2009-2017 Facebook, Inc. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
1. Redistributions of source code must retain the above copyright notice(s),
|
||||
this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice(s),
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
|
||||
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||
EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
||||
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
--------------------------------------------------------------------------------
|
||||
@@ -0,0 +1,60 @@
|
||||
<!---
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Red Arrow GPU - Apache Arrow GPU Ruby
|
||||
|
||||
Red Arrow GPU is the Ruby bindings of Apache Arrow GPU. Red Arrow GPU is based on GObject Introspection.
|
||||
|
||||
[Apache Arrow GPU](https://arrow.apache.org/) is an in-memory columnar data store on GPU.
|
||||
|
||||
[GObject Introspection](https://wiki.gnome.org/action/show/Projects/GObjectIntrospection) is a middleware for language bindings of C library. GObject Introspection can generate language bindings automatically at runtime.
|
||||
|
||||
Red Arrow GPU uses [Apache Arrow GPU GLib](https://github.com/apache/arrow/tree/master/c_glib) and [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection) to generate Ruby bindings of Apache Arrow GPU.
|
||||
|
||||
Apache Arrow GPU GLib is a C wrapper for [Apache Arrow GPU C++](https://github.com/apache/arrow/tree/master/cpp). GObject Introspection can't use Apache Arrow GPU C++ directly. Apache Arrow GPU GLib is a bridge between Apache Arrow GPU C++ and GObject Introspection.
|
||||
|
||||
gobject-introspection gem is a Ruby bindings of GObject Introspection. Red Arrow GPU uses GObject Introspection via gobject-introspection gem.
|
||||
|
||||
## Install
|
||||
|
||||
Install Apache Arrow GPU GLib before install Red Arrow GPU. Use [packages.red-data-tools.org](https://github.com/red-data-tools/packages.red-data-tools.org) for installing Apache Arrow GPU GLib.
|
||||
|
||||
Install Red Arrow GPU after you install Apache Arrow GPU GLib:
|
||||
|
||||
```text
|
||||
% gem install red-arrow-gpu
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```ruby
|
||||
require "arrow-gpu"
|
||||
|
||||
manager = ArrowGPU::CUDADeviceManager.new
|
||||
if manager.n_devices.zero?
|
||||
raise "No GPU is found"
|
||||
end
|
||||
|
||||
context = manager[0]
|
||||
buffer = ArrowGPU::CUDABuffer.new(context, 128)
|
||||
ArrowGPU::CUDABufferOutputStream.open(buffer) do |stream|
|
||||
stream.write("Hello World")
|
||||
end
|
||||
puts buffer.copy_to_host(0, 11) # => "Hello World"
|
||||
```
|
||||
@@ -0,0 +1,41 @@
|
||||
# -*- ruby -*-
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "rubygems"
|
||||
require "bundler/gem_helper"
|
||||
|
||||
base_dir = File.join(File.dirname(__FILE__))
|
||||
|
||||
helper = Bundler::GemHelper.new(base_dir)
|
||||
def helper.version_tag
|
||||
version
|
||||
end
|
||||
|
||||
helper.install
|
||||
spec = helper.gemspec
|
||||
|
||||
desc "Run tests"
|
||||
task :test do
|
||||
cd("dependency-check") do
|
||||
ruby("-S", "rake")
|
||||
end
|
||||
ruby("test/run-test.rb")
|
||||
end
|
||||
|
||||
task default: :test
|
||||
@@ -0,0 +1,43 @@
|
||||
# -*- ruby -*-
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "pkg-config"
|
||||
require "native-package-installer"
|
||||
|
||||
case RUBY_PLATFORM
|
||||
when /mingw|mswin/
|
||||
task :default => "nothing"
|
||||
else
|
||||
task :default => "dependency:check"
|
||||
end
|
||||
|
||||
task :nothing do
|
||||
end
|
||||
|
||||
namespace :dependency do
|
||||
desc "Check dependency"
|
||||
task :check do
|
||||
unless PKGConfig.check_version?("arrow-gpu-glib")
|
||||
unless NativePackageInstaller.install(:debian => "libarrow-gpu-glib-dev",
|
||||
:redhat => "arrow-gpu-glib-devel")
|
||||
exit(false)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,29 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "arrow"
|
||||
|
||||
require "arrow-gpu/version"
|
||||
|
||||
require "arrow-gpu/loader"
|
||||
|
||||
module ArrowGPU
|
||||
class Error < StandardError
|
||||
end
|
||||
|
||||
Loader.load
|
||||
end
|
||||
@@ -0,0 +1,25 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module ArrowGPU
|
||||
class CUDADeviceManager
|
||||
# Experimental.
|
||||
#
|
||||
# Can we think device manager is a container of contexts?
|
||||
alias_method :[], :get_context
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,35 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module ArrowGPU
|
||||
class Loader < GObjectIntrospection::Loader
|
||||
class << self
|
||||
def load
|
||||
super("ArrowGPU", ArrowGPU)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def post_load(repository, namespace)
|
||||
require_libraries
|
||||
end
|
||||
|
||||
def require_libraries
|
||||
require "arrow-gpu/cuda-device-manager"
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,50 @@
|
||||
# -*- ruby -*-
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require_relative "version"
|
||||
|
||||
Gem::Specification.new do |spec|
|
||||
spec.name = "red-arrow-gpu"
|
||||
version_components = [
|
||||
ArrowGPU::Version::MAJOR.to_s,
|
||||
ArrowGPU::Version::MINOR.to_s,
|
||||
ArrowGPU::Version::MICRO.to_s,
|
||||
# "beta1",
|
||||
]
|
||||
spec.version = version_components.join(".")
|
||||
spec.homepage = "https://arrow.apache.org/"
|
||||
spec.authors = ["Apache Arrow Developers"]
|
||||
spec.email = ["dev@arrow.apache.org"]
|
||||
|
||||
spec.summary = "Red Arrow GPU is the Ruby bindings of Apache Arrow GPU"
|
||||
spec.description =
|
||||
"Apache Arrow GPU is a common in-memory columnar data store on GPU. " +
|
||||
"It's useful to share and process large data."
|
||||
spec.license = "Apache-2.0"
|
||||
spec.files = ["README.md", "Rakefile", "Gemfile", "#{spec.name}.gemspec"]
|
||||
spec.files += Dir.glob("lib/**/*.rb")
|
||||
spec.test_files += Dir.glob("test/**/*")
|
||||
spec.extensions = ["dependency-check/Rakefile"]
|
||||
|
||||
spec.add_runtime_dependency("red-arrow")
|
||||
|
||||
spec.add_development_dependency("bundler")
|
||||
spec.add_development_dependency("rake")
|
||||
spec.add_development_dependency("test-unit")
|
||||
end
|
||||
@@ -0,0 +1,22 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require_relative "../version"
|
||||
|
||||
require "arrow-gpu"
|
||||
|
||||
require "test-unit"
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env ruby
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
$VERBOSE = true
|
||||
|
||||
require "pathname"
|
||||
|
||||
base_dir = Pathname.new(__dir__).parent.expand_path
|
||||
arrow_base_dir = base_dir.parent + "red-arrow"
|
||||
|
||||
lib_dir = base_dir + "lib"
|
||||
test_dir = base_dir + "test"
|
||||
|
||||
arrow_lib_dir = arrow_base_dir + "lib"
|
||||
|
||||
$LOAD_PATH.unshift(arrow_lib_dir.to_s)
|
||||
$LOAD_PATH.unshift(lib_dir.to_s)
|
||||
|
||||
require_relative "helper"
|
||||
|
||||
exit(Test::Unit::AutoRunner.run(true, test_dir.to_s))
|
||||
@@ -0,0 +1,38 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class TestCUDA < Test::Unit::TestCase
|
||||
def setup
|
||||
@manager = ArrowGPU::CUDADeviceManager.new
|
||||
omit("At least one GPU is required") if @manager.n_devices.zero?
|
||||
@context = @manager[0]
|
||||
end
|
||||
|
||||
sub_test_case("BufferOutputStream") do
|
||||
def setup
|
||||
super
|
||||
@buffer = ArrowGPU::CUDABuffer.new(@context, 128)
|
||||
end
|
||||
|
||||
def test_new
|
||||
ArrowGPU::CUDABufferOutputStream.open(@buffer) do |stream|
|
||||
stream.write("Hello World")
|
||||
end
|
||||
assert_equal("Hello World", @buffer.copy_to_host(0, 11).to_s)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,71 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "pathname"
|
||||
|
||||
version_rb_path = Pathname.new(__FILE__)
|
||||
base_dir = version_rb_path.dirname
|
||||
pom_xml_path = base_dir.join("..", "..", "java", "pom.xml")
|
||||
lib_version_rb_path = base_dir.join("lib", "arrow-gpu", "version.rb")
|
||||
|
||||
need_update = false
|
||||
if not lib_version_rb_path.exist?
|
||||
need_update = true
|
||||
elsif version_rb_path.mtime > lib_version_rb_path.mtime
|
||||
need_update = true
|
||||
elsif pom_xml_path.exist? and pom_xml_path.mtime > lib_version_rb_path.mtime
|
||||
need_update = true
|
||||
end
|
||||
|
||||
if need_update
|
||||
version = pom_xml_path.read.scan(/^ <version>(.+?)<\/version>/)[0][0]
|
||||
major, minor, micro, tag = version.split(/[.-]/)
|
||||
lib_version_rb_path.open("w") do |lib_version_rb|
|
||||
lib_version_rb.puts(<<-RUBY)
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module ArrowGPU
|
||||
module Version
|
||||
MAJOR = #{major}
|
||||
MINOR = #{minor}
|
||||
MICRO = #{micro}
|
||||
TAG = #{tag ? tag.dump : nil}
|
||||
STRING = #{version.dump}
|
||||
end
|
||||
|
||||
VERSION = Version::STRING
|
||||
end
|
||||
RUBY
|
||||
end
|
||||
end
|
||||
|
||||
require_relative "lib/arrow-gpu/version"
|
||||
@@ -0,0 +1,20 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
/lib/arrow/version.rb
|
||||
|
||||
/pkg/
|
||||
@@ -0,0 +1,22 @@
|
||||
# -*- ruby -*-
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
source "https://rubygems.org/"
|
||||
|
||||
gemspec
|
||||
@@ -0,0 +1,585 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/plasma/fling.cc and src/plasma/fling.h: Apache 2.0
|
||||
|
||||
Copyright 2013 Sharvil Nanavati
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/plasma/thirdparty/ae: Modified / 3-Clause BSD
|
||||
|
||||
Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
* Neither the name of Redis nor the names of its contributors may be used
|
||||
to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/plasma/thirdparty/dlmalloc.c: CC0
|
||||
|
||||
This is a version (aka dlmalloc) of malloc/free/realloc written by
|
||||
Doug Lea and released to the public domain, as explained at
|
||||
http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
|
||||
comments, complaints, performance data, etc to dl@cs.oswego.edu
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/plasma/thirdparty/xxhash: BSD 2-Clause License
|
||||
|
||||
xxHash - Fast Hash algorithm
|
||||
Copyright (C) 2012-2016, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- xxHash homepage: http://www.xxhash.com
|
||||
- xxHash source repository : https://github.com/Cyan4973/xxHash
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
src/arrow/util (some portions): Apache 2.0, and 3-clause BSD
|
||||
|
||||
Some portions of this module are derived from code in the Chromium project,
|
||||
copyright (c) Google inc and (c) The Chromium Authors and licensed under the
|
||||
Apache 2.0 License or the under the 3-clause BSD license:
|
||||
|
||||
Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from Daniel Lemire's FrameOfReference project.
|
||||
|
||||
https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp
|
||||
|
||||
Copyright: 2013 Daniel Lemire
|
||||
Home page: http://lemire.me/en/
|
||||
Project page: https://github.com/lemire/FrameOfReference
|
||||
License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the TensorFlow project
|
||||
|
||||
Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the NumPy project.
|
||||
|
||||
https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
|
||||
|
||||
https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c
|
||||
|
||||
Copyright (c) 2005-2017, NumPy Developers.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following
|
||||
disclaimer in the documentation and/or other materials provided
|
||||
with the distribution.
|
||||
|
||||
* Neither the name of the NumPy Developers nor the names of any
|
||||
contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the Boost project
|
||||
|
||||
Boost Software License - Version 1.0 - August 17th, 2003
|
||||
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the mapbox/variant project, BSD 3-clause
|
||||
license
|
||||
|
||||
Copyright (c) MapBox
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
- Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
- Redistributions in binary form must reproduce the above copyright notice, this
|
||||
list of conditions and the following disclaimer in the documentation and/or
|
||||
other materials provided with the distribution.
|
||||
- Neither the name "MapBox" nor the names of its contributors may be
|
||||
used to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the Boost project
|
||||
|
||||
Boost Software License - Version 1.0 - August 17th, 2003
|
||||
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the mapbox/variant project, BSD 3-clause
|
||||
license
|
||||
|
||||
Copyright (c) MapBox
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
- Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
- Redistributions in binary form must reproduce the above copyright notice, this
|
||||
list of conditions and the following disclaimer in the documentation and/or
|
||||
other materials provided with the distribution.
|
||||
- Neither the name "MapBox" nor the names of its contributors may be
|
||||
used to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the FlatBuffers project
|
||||
|
||||
Copyright 2014 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the tslib project
|
||||
|
||||
Copyright 2015 Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
This project includes code from the jemalloc project
|
||||
|
||||
https://github.com/jemalloc/jemalloc
|
||||
|
||||
Copyright (C) 2002-2017 Jason Evans <jasone@canonware.com>.
|
||||
All rights reserved.
|
||||
Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
|
||||
Copyright (C) 2009-2017 Facebook, Inc. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
1. Redistributions of source code must retain the above copyright notice(s),
|
||||
this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice(s),
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
|
||||
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||
EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
||||
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
--------------------------------------------------------------------------------
|
||||
@@ -0,0 +1,52 @@
|
||||
<!---
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Red Arrow - Apache Arrow Ruby
|
||||
|
||||
Red Arrow is the Ruby bindings of Apache Arrow. Red Arrow is based on GObject Introspection.
|
||||
|
||||
[Apache Arrow](https://arrow.apache.org/) is an in-memory columnar data store. It's used by many products for data analytics.
|
||||
|
||||
[GObject Introspection](https://wiki.gnome.org/action/show/Projects/GObjectIntrospection) is a middleware for language bindings of C library. GObject Introspection can generate language bindings automatically at runtime.
|
||||
|
||||
Red Arrow uses [Apache Arrow GLib](https://github.com/apache/arrow/tree/master/c_glib) and [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection) to generate Ruby bindings of Apache Arrow.
|
||||
|
||||
Apache Arrow GLib is a C wrapper for [Apache Arrow C++](https://github.com/apache/arrow/tree/master/cpp). GObject Introspection can't use Apache Arrow C++ directly. Apache Arrow GLib is a bridge between Apache Arrow C++ and GObject Introspection.
|
||||
|
||||
gobject-introspection gem is a Ruby bindings of GObject Introspection. Red Arrow uses GObject Introspection via gobject-introspection gem.
|
||||
|
||||
## Install
|
||||
|
||||
Install Apache Arrow GLib before install Red Arrow. Use [packages.red-data-tools.org](https://github.com/red-data-tools/packages.red-data-tools.org) for installing Apache Arrow GLib.
|
||||
|
||||
Install Red Arrow after you install Apache Arrow GLib:
|
||||
|
||||
```text
|
||||
% gem install red-arrow
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```ruby
|
||||
require "arrow"
|
||||
|
||||
table = Arrow::Table.load("/dev/shm/data.arrow")
|
||||
# Process data in table
|
||||
table.save("/dev/shm/data-processed.arrow")
|
||||
```
|
||||
@@ -0,0 +1,41 @@
|
||||
# -*- ruby -*-
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "rubygems"
|
||||
require "bundler/gem_helper"
|
||||
|
||||
base_dir = File.join(__dir__)
|
||||
|
||||
helper = Bundler::GemHelper.new(base_dir)
|
||||
def helper.version_tag
|
||||
version
|
||||
end
|
||||
|
||||
helper.install
|
||||
spec = helper.gemspec
|
||||
|
||||
desc "Run tests"
|
||||
task :test do
|
||||
cd("dependency-check") do
|
||||
ruby("-S", "rake")
|
||||
end
|
||||
ruby("test/run-test.rb")
|
||||
end
|
||||
|
||||
task default: :test
|
||||
@@ -0,0 +1,43 @@
|
||||
# -*- ruby -*-
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "pkg-config"
|
||||
require "native-package-installer"
|
||||
|
||||
case RUBY_PLATFORM
|
||||
when /mingw|mswin/
|
||||
task :default => "nothing"
|
||||
else
|
||||
task :default => "dependency:check"
|
||||
end
|
||||
|
||||
task :nothing do
|
||||
end
|
||||
|
||||
namespace :dependency do
|
||||
desc "Check dependency"
|
||||
task :check do
|
||||
unless PKGConfig.check_version?("arrow-glib", 0, 9, 0)
|
||||
unless NativePackageInstaller.install(:debian => "libarrow-glib-dev",
|
||||
:redhat => "arrow-glib-devel")
|
||||
exit(false)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,34 @@
|
||||
<!---
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Development
|
||||
|
||||
## Naming convention
|
||||
|
||||
### Reader and Writer
|
||||
|
||||
Reader and Writer require an opened IO stream.
|
||||
|
||||
### Loader and Saver
|
||||
|
||||
Loader and Saver require a path. They are convenient classes.
|
||||
|
||||
Loader opens the path and reads data by Reader.
|
||||
|
||||
Writer opens the path and writes data by Writer.
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env ruby
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "arrow"
|
||||
|
||||
Arrow::MemoryMappedInputStream.open("/tmp/file.arrow") do |input|
|
||||
reader = Arrow::RecordBatchFileReader.new(input)
|
||||
fields = reader.schema.fields
|
||||
reader.each_with_index do |record_batch, i|
|
||||
puts("=" * 40)
|
||||
puts("record-batch[#{i}]:")
|
||||
fields.each do |field|
|
||||
field_name = field.name
|
||||
values = record_batch.collect do |record|
|
||||
record[field_name]
|
||||
end
|
||||
puts(" #{field_name}: #{values.inspect}")
|
||||
end
|
||||
end
|
||||
end
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env ruby
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "arrow"
|
||||
|
||||
Arrow::MemoryMappedInputStream.open("/tmp/stream.arrow") do |input|
|
||||
reader = Arrow::RecordBatchStreamReader.new(input)
|
||||
fields = reader.schema.fields
|
||||
reader.each_with_index do |record_batch, i|
|
||||
puts("=" * 40)
|
||||
puts("record-batch[#{i}]:")
|
||||
fields.each do |field|
|
||||
field_name = field.name
|
||||
values = record_batch.collect do |record|
|
||||
record[field_name]
|
||||
end
|
||||
puts(" #{field_name}: #{values.inspect}")
|
||||
end
|
||||
end
|
||||
end
|
||||
Executable
+63
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env ruby
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "arrow"
|
||||
|
||||
fields = [
|
||||
Arrow::Field.new("uint8", :uint8),
|
||||
Arrow::Field.new("uint16", :uint16),
|
||||
Arrow::Field.new("uint32", :uint32),
|
||||
Arrow::Field.new("uint64", :uint64),
|
||||
Arrow::Field.new("int8", :int8),
|
||||
Arrow::Field.new("int16", :int16),
|
||||
Arrow::Field.new("int32", :int32),
|
||||
Arrow::Field.new("int64", :int64),
|
||||
Arrow::Field.new("float", :float),
|
||||
Arrow::Field.new("double", :double),
|
||||
]
|
||||
schema = Arrow::Schema.new(fields)
|
||||
|
||||
Arrow::FileOutputStream.open("/tmp/file.arrow", false) do |output|
|
||||
Arrow::RecordBatchFileWriter.open(output, schema) do |writer|
|
||||
uints = [1, 2, 4, 8]
|
||||
ints = [1, -2, 4, -8]
|
||||
floats = [1.1, -2.2, 4.4, -8.8]
|
||||
columns = [
|
||||
Arrow::UInt8Array.new(uints),
|
||||
Arrow::UInt16Array.new(uints),
|
||||
Arrow::UInt32Array.new(uints),
|
||||
Arrow::UInt64Array.new(uints),
|
||||
Arrow::Int8Array.new(ints),
|
||||
Arrow::Int16Array.new(ints),
|
||||
Arrow::Int32Array.new(ints),
|
||||
Arrow::Int64Array.new(ints),
|
||||
Arrow::FloatArray.new(floats),
|
||||
Arrow::DoubleArray.new(floats),
|
||||
]
|
||||
|
||||
record_batch = Arrow::RecordBatch.new(schema, 4, columns)
|
||||
writer.write_record_batch(record_batch)
|
||||
|
||||
sliced_columns = columns.collect do |column|
|
||||
column.slice(1, 3)
|
||||
end
|
||||
record_batch = Arrow::RecordBatch.new(schema, 3, sliced_columns)
|
||||
writer.write_record_batch(record_batch)
|
||||
end
|
||||
end
|
||||
Executable
+63
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env ruby
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "arrow"
|
||||
|
||||
fields = [
|
||||
Arrow::Field.new("uint8", :uint8),
|
||||
Arrow::Field.new("uint16", :uint16),
|
||||
Arrow::Field.new("uint32", :uint32),
|
||||
Arrow::Field.new("uint64", :uint64),
|
||||
Arrow::Field.new("int8", :int8),
|
||||
Arrow::Field.new("int16", :int16),
|
||||
Arrow::Field.new("int32", :int32),
|
||||
Arrow::Field.new("int64", :int64),
|
||||
Arrow::Field.new("float", :float),
|
||||
Arrow::Field.new("double", :double),
|
||||
]
|
||||
schema = Arrow::Schema.new(fields)
|
||||
|
||||
Arrow::FileOutputStream.open("/tmp/stream.arrow", false) do |output|
|
||||
Arrow::RecordBatchStreamWriter.open(output, schema) do |writer|
|
||||
uints = [1, 2, 4, 8]
|
||||
ints = [1, -2, 4, -8]
|
||||
floats = [1.1, -2.2, 4.4, -8.8]
|
||||
columns = [
|
||||
Arrow::UInt8Array.new(uints),
|
||||
Arrow::UInt16Array.new(uints),
|
||||
Arrow::UInt32Array.new(uints),
|
||||
Arrow::UInt64Array.new(uints),
|
||||
Arrow::Int8Array.new(ints),
|
||||
Arrow::Int16Array.new(ints),
|
||||
Arrow::Int32Array.new(ints),
|
||||
Arrow::Int64Array.new(ints),
|
||||
Arrow::FloatArray.new(floats),
|
||||
Arrow::DoubleArray.new(floats),
|
||||
]
|
||||
|
||||
record_batch = Arrow::RecordBatch.new(schema, 4, columns)
|
||||
writer.write_record_batch(record_batch)
|
||||
|
||||
sliced_columns = columns.collect do |column|
|
||||
column.slice(1, 3)
|
||||
end
|
||||
record_batch = Arrow::RecordBatch.new(schema, 3, sliced_columns)
|
||||
writer.write_record_batch(record_batch)
|
||||
end
|
||||
end
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 7.0 KiB |
@@ -0,0 +1,29 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "gobject-introspection"
|
||||
|
||||
require "arrow/version"
|
||||
|
||||
require "arrow/loader"
|
||||
|
||||
module Arrow
|
||||
class Error < StandardError
|
||||
end
|
||||
|
||||
Loader.load
|
||||
end
|
||||
@@ -0,0 +1,124 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "date"
|
||||
|
||||
module Arrow
|
||||
class ArrayBuilder
|
||||
class << self
|
||||
def build(values)
|
||||
if self != ArrayBuilder
|
||||
builder = new
|
||||
return builder.build(values)
|
||||
end
|
||||
|
||||
builder_class = nil
|
||||
values.each do |value|
|
||||
case value
|
||||
when nil
|
||||
# Ignore
|
||||
when true, false
|
||||
return BooleanArray.new(values)
|
||||
when String
|
||||
return StringArray.new(values)
|
||||
when Float
|
||||
return DoubleArray.new(values)
|
||||
when Integer
|
||||
if value < 0
|
||||
builder = IntArrayBuilder.new
|
||||
return builder.build(values)
|
||||
else
|
||||
builder_class = UIntArrayBuilder
|
||||
end
|
||||
when Time
|
||||
data_type = TimestampDataType.new(:nano)
|
||||
builder = TimestampArrayBuilder.new(data_type)
|
||||
return builder.build(values)
|
||||
when DateTime
|
||||
return Date64Array.new(values)
|
||||
when Date
|
||||
return Date32Array.new(values)
|
||||
else
|
||||
return StringArray.new(values)
|
||||
end
|
||||
end
|
||||
if builder_class
|
||||
builder_class.new.build(values)
|
||||
else
|
||||
Arrow::StringArray.new(values)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def build(values)
|
||||
value_convertable = respond_to?(:convert_to_arrow_value, true)
|
||||
start_index = 0
|
||||
current_index = 0
|
||||
status = :value
|
||||
|
||||
values.each do |value|
|
||||
if value.nil?
|
||||
if status == :value
|
||||
if start_index != current_index
|
||||
target_values = values[start_index...current_index]
|
||||
if value_convertable
|
||||
target_values = target_values.collect do |v|
|
||||
convert_to_arrow_value(v)
|
||||
end
|
||||
end
|
||||
append_values(target_values, nil)
|
||||
start_index = current_index
|
||||
end
|
||||
status = :null
|
||||
end
|
||||
else
|
||||
if status == :null
|
||||
append_nulls(current_index - start_index)
|
||||
start_index = current_index
|
||||
status = :value
|
||||
end
|
||||
end
|
||||
current_index += 1
|
||||
end
|
||||
if start_index != current_index
|
||||
if status == :value
|
||||
if start_index == 0 and current_index == values.size
|
||||
target_values = values
|
||||
else
|
||||
target_values = values[start_index...current_index]
|
||||
end
|
||||
if value_convertable
|
||||
target_values = target_values.collect do |v|
|
||||
convert_to_arrow_value(v)
|
||||
end
|
||||
end
|
||||
append_values(target_values, nil)
|
||||
else
|
||||
append_nulls(current_index - start_index)
|
||||
end
|
||||
end
|
||||
|
||||
finish
|
||||
end
|
||||
|
||||
def append_nulls(n)
|
||||
n.times do
|
||||
append_null
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,63 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class Array
|
||||
include Enumerable
|
||||
|
||||
class << self
|
||||
def new(values)
|
||||
builder_class_name = "#{name}Builder"
|
||||
if const_defined?(builder_class_name)
|
||||
builder_class = const_get(builder_class_name)
|
||||
builder_class.build(values)
|
||||
else
|
||||
super
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def [](i)
|
||||
i += length if i < 0
|
||||
if null?(i)
|
||||
nil
|
||||
else
|
||||
get_value(i)
|
||||
end
|
||||
end
|
||||
|
||||
def each
|
||||
return to_enum(__method__) unless block_given?
|
||||
|
||||
length.times do |i|
|
||||
yield(self[i])
|
||||
end
|
||||
end
|
||||
|
||||
def reverse_each
|
||||
return to_enum(__method__) unless block_given?
|
||||
|
||||
(length - 1).downto(0) do |i|
|
||||
yield(self[i])
|
||||
end
|
||||
end
|
||||
|
||||
def to_arrow
|
||||
self
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,31 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
module BlockClosable
|
||||
def open(*args, &block)
|
||||
io = new(*args)
|
||||
return io unless block
|
||||
|
||||
begin
|
||||
yield(io)
|
||||
ensure
|
||||
io.close
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,84 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class ChunkedArray
|
||||
include Enumerable
|
||||
|
||||
alias_method :chunks_raw, :chunks
|
||||
def chunks
|
||||
@chunks ||= chunks_raw
|
||||
end
|
||||
|
||||
def null?(i)
|
||||
chunks.each do |array|
|
||||
return array.null?(i) if i < array.length
|
||||
i -= array.length
|
||||
end
|
||||
nil
|
||||
end
|
||||
|
||||
def valid?(i)
|
||||
chunks.each do |array|
|
||||
return array.valid?(i) if i < array.length
|
||||
i -= array.length
|
||||
end
|
||||
nil
|
||||
end
|
||||
|
||||
def [](i)
|
||||
i += length if i < 0
|
||||
chunks.each do |array|
|
||||
return array[i] if i < array.length
|
||||
i -= array.length
|
||||
end
|
||||
nil
|
||||
end
|
||||
|
||||
def each(&block)
|
||||
return to_enum(__method__) unless block_given?
|
||||
|
||||
chunks.each do |array|
|
||||
array.each(&block)
|
||||
end
|
||||
end
|
||||
|
||||
def reverse_each(&block)
|
||||
return to_enum(__method__) unless block_given?
|
||||
|
||||
chunks.reverse_each do |array|
|
||||
array.reverse_each(&block)
|
||||
end
|
||||
end
|
||||
|
||||
def each_chunk(&block)
|
||||
chunks.each(&block)
|
||||
end
|
||||
|
||||
def pack
|
||||
first_chunk = chunks.first
|
||||
data_type = first_chunk.value_data_type
|
||||
case data_type
|
||||
when TimestampDataType
|
||||
builder = TimestampArrayBuilder.new(data_type)
|
||||
builder.build(to_a)
|
||||
else
|
||||
first_chunk.class.new(to_a)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,50 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class Column
|
||||
include Enumerable
|
||||
|
||||
def null?(i)
|
||||
data.null?(i)
|
||||
end
|
||||
|
||||
def valid?(i)
|
||||
data.valid?(i)
|
||||
end
|
||||
|
||||
def [](i)
|
||||
data[i]
|
||||
end
|
||||
|
||||
def each(&block)
|
||||
return to_enum(__method__) unless block_given?
|
||||
|
||||
data.each(&block)
|
||||
end
|
||||
|
||||
def reverse_each(&block)
|
||||
return to_enum(__method__) unless block_given?
|
||||
|
||||
data.reverse_each(&block)
|
||||
end
|
||||
|
||||
def pack
|
||||
self.class.new(field, data.pack)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,250 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "csv"
|
||||
require "pathname"
|
||||
require "time"
|
||||
|
||||
module Arrow
|
||||
class CSVLoader
|
||||
class << self
|
||||
def load(path_or_data, **options)
|
||||
new(path_or_data, **options).load
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(path_or_data, **options)
|
||||
@path_or_data = path_or_data
|
||||
@options = options
|
||||
end
|
||||
|
||||
def load
|
||||
case @path_or_data
|
||||
when Pathname
|
||||
load_from_path(@path_or_data.to_path)
|
||||
when /\A.+\.csv\z/i
|
||||
load_from_path(@path_or_data)
|
||||
else
|
||||
load_data(@path_or_data)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def open_csv(path, **options)
|
||||
CSV.open(path, **options) do |csv|
|
||||
yield(csv)
|
||||
end
|
||||
end
|
||||
|
||||
def parse_csv_data(data, **options)
|
||||
csv = CSV.new(data, **options)
|
||||
begin
|
||||
yield(csv)
|
||||
ensure
|
||||
csv.close
|
||||
end
|
||||
end
|
||||
|
||||
def read_csv(csv)
|
||||
reader = CSVReader.new(csv)
|
||||
reader.read
|
||||
end
|
||||
|
||||
def load_from_path(path)
|
||||
options = update_csv_parse_options(@options, :open_csv, path)
|
||||
open_csv(path, **options) do |csv|
|
||||
read_csv(csv)
|
||||
end
|
||||
end
|
||||
|
||||
def load_data(data)
|
||||
options = update_csv_parse_options(@options, :parse_csv_data, data)
|
||||
parse_csv_data(data, **options) do |csv|
|
||||
read_csv(csv)
|
||||
end
|
||||
end
|
||||
|
||||
def selective_converter(target_index)
|
||||
lambda do |field, field_info|
|
||||
if target_index.nil? or field_info.index == target_index
|
||||
yield(field)
|
||||
else
|
||||
field
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
BOOLEAN_CONVERTER = lambda do |field|
|
||||
begin
|
||||
encoded_field = field.encode(CSV::ConverterEncoding)
|
||||
rescue EncodingError
|
||||
field
|
||||
else
|
||||
case encoded_field
|
||||
when "true"
|
||||
true
|
||||
when "false"
|
||||
false
|
||||
else
|
||||
field
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
ISO8601_CONVERTER = lambda do |field|
|
||||
begin
|
||||
encoded_field = field.encode(CSV::ConverterEncoding)
|
||||
rescue EncodingError
|
||||
field
|
||||
else
|
||||
begin
|
||||
Time.iso8601(encoded_field)
|
||||
rescue ArgumentError
|
||||
field
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def update_csv_parse_options(options, create_csv, *args)
|
||||
if options.key?(:converters)
|
||||
new_options = options.dup
|
||||
else
|
||||
converters = [:all, BOOLEAN_CONVERTER, ISO8601_CONVERTER]
|
||||
new_options = options.merge(converters: converters)
|
||||
end
|
||||
|
||||
unless options.key?(:headers)
|
||||
__send__(create_csv, *args, **new_options) do |csv|
|
||||
new_options[:headers] = have_header?(csv)
|
||||
end
|
||||
end
|
||||
unless options.key?(:converters)
|
||||
__send__(create_csv, *args, **new_options) do |csv|
|
||||
new_options[:converters] = detect_robust_converters(csv)
|
||||
end
|
||||
end
|
||||
|
||||
new_options
|
||||
end
|
||||
|
||||
def have_header?(csv)
|
||||
if @options.key?(:headers)
|
||||
return @options[:headers]
|
||||
end
|
||||
|
||||
row1 = csv.shift
|
||||
return false if row1.nil?
|
||||
return false if row1.any?(&:nil?)
|
||||
|
||||
row2 = csv.shift
|
||||
return nil if row2.nil?
|
||||
return true if row2.any?(&:nil?)
|
||||
|
||||
return false if row1.any? {|value| not value.is_a?(String)}
|
||||
|
||||
if row1.collect(&:class) != row2.collect(&:class)
|
||||
return true
|
||||
end
|
||||
|
||||
nil
|
||||
end
|
||||
|
||||
def detect_robust_converters(csv)
|
||||
column_types = []
|
||||
csv.each do |row|
|
||||
if row.is_a?(CSV::Row)
|
||||
each_value = Enumerator.new do |yielder|
|
||||
row.each do |_name, value|
|
||||
yielder << value
|
||||
end
|
||||
end
|
||||
else
|
||||
each_value = row.each
|
||||
end
|
||||
each_value.with_index do |value, i|
|
||||
current_column_type = column_types[i]
|
||||
next if current_column_type == :string
|
||||
|
||||
candidate_type = nil
|
||||
case value
|
||||
when nil
|
||||
next
|
||||
when "true", "false", true, false
|
||||
candidate_type = :boolean
|
||||
when Integer
|
||||
candidate_type = :integer
|
||||
if current_column_type == :float
|
||||
candidate_type = :float
|
||||
end
|
||||
when Float
|
||||
candidate_type = :float
|
||||
if current_column_type == :integer
|
||||
column_types[i] = candidate_type
|
||||
end
|
||||
when Time
|
||||
candidate_type = :time
|
||||
when DateTime
|
||||
candidate_type = :date_time
|
||||
when Date
|
||||
candidate_type = :date
|
||||
when String
|
||||
next if value.empty?
|
||||
candidate_type = :string
|
||||
else
|
||||
candidate_type = :string
|
||||
end
|
||||
|
||||
column_types[i] ||= candidate_type
|
||||
if column_types[i] != candidate_type
|
||||
column_types[i] = :string
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
converters = []
|
||||
column_types.each_with_index do |type, i|
|
||||
case type
|
||||
when :boolean
|
||||
converters << selective_converter(i, &BOOLEAN_CONVERTER)
|
||||
when :integer
|
||||
converters << selective_converter(i) do |field|
|
||||
if field.nil? or field.empty?
|
||||
nil
|
||||
else
|
||||
CSV::Converters[:integer].call(field)
|
||||
end
|
||||
end
|
||||
when :float
|
||||
converters << selective_converter(i) do |field|
|
||||
if field.nil? or field.empty?
|
||||
nil
|
||||
else
|
||||
CSV::Converters[:float].call(field)
|
||||
end
|
||||
end
|
||||
when :time
|
||||
converters << selective_converter(i, &ISO8601_CONVERTER)
|
||||
when :date_time
|
||||
converters << selective_converter(i, &CSV::Converters[:date_time])
|
||||
when :date
|
||||
converters << selective_converter(i, &CSV::Converters[:date])
|
||||
end
|
||||
end
|
||||
converters
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,54 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "csv"
|
||||
|
||||
module Arrow
|
||||
class CSVReader
|
||||
def initialize(csv)
|
||||
@csv = csv
|
||||
end
|
||||
|
||||
def read
|
||||
values_set = []
|
||||
@csv.each do |row|
|
||||
if row.is_a?(CSV::Row)
|
||||
row = row.collect(&:last)
|
||||
end
|
||||
row.each_with_index do |value, i|
|
||||
values = (values_set[i] ||= [])
|
||||
values << value
|
||||
end
|
||||
end
|
||||
return nil if values_set.empty?
|
||||
|
||||
arrays = values_set.collect.with_index do |values, i|
|
||||
ArrayBuilder.build(values)
|
||||
end
|
||||
if @csv.headers
|
||||
names = @csv.headers
|
||||
else
|
||||
names = arrays.size.times.collect(&:to_s)
|
||||
end
|
||||
raw_table = {}
|
||||
names.each_with_index do |name, i|
|
||||
raw_table[name] = arrays[i]
|
||||
end
|
||||
Table.new(raw_table)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,32 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class Date32ArrayBuilder
|
||||
private
|
||||
UNIX_EPOCH = Date.new(1970, 1, 1)
|
||||
def convert_to_arrow_value(value)
|
||||
value = value.to_date if value.respond_to?(:to_date)
|
||||
|
||||
if value.is_a?(Date)
|
||||
(value - UNIX_EPOCH).to_i
|
||||
else
|
||||
value
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,30 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class Date32Array
|
||||
def get_value(i)
|
||||
to_date(get_raw_value(i))
|
||||
end
|
||||
|
||||
private
|
||||
UNIX_EPOCH = 2440588
|
||||
def to_date(raw_value)
|
||||
Date.jd(UNIX_EPOCH + raw_value)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,33 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class Date64ArrayBuilder
|
||||
private
|
||||
def convert_to_arrow_value(value)
|
||||
if value.respond_to?(:to_time) and not value.is_a?(Time)
|
||||
value = value.to_time
|
||||
end
|
||||
|
||||
if value.is_a?(Time)
|
||||
value.to_i * 1_000 + value.usec / 1_000
|
||||
else
|
||||
value
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,29 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class Date64Array
|
||||
def get_value(i)
|
||||
to_datetime(get_raw_value(i))
|
||||
end
|
||||
|
||||
private
|
||||
def to_datetime(raw_value)
|
||||
Time.at(*raw_value.divmod(1_000)).to_datetime
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,34 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class Field
|
||||
alias_method :initialize_raw, :initialize
|
||||
def initialize(name, data_type)
|
||||
case data_type
|
||||
when String, Symbol
|
||||
data_type_name = data_type.to_s.capitalize.gsub(/\AUint/, "UInt")
|
||||
data_type_class_name = "#{data_type_name}DataType"
|
||||
if Arrow.const_defined?(data_type_class_name)
|
||||
data_type_class = Arrow.const_get(data_type_class_name)
|
||||
data_type = data_type_class.new
|
||||
end
|
||||
end
|
||||
initialize_raw(name, data_type)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,175 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
# Experimental
|
||||
#
|
||||
# TODO: Almost codes should be implemented in Apache Arrow C++.
|
||||
class Group
|
||||
def initialize(table, keys)
|
||||
@table = table
|
||||
@keys = keys
|
||||
end
|
||||
|
||||
def count
|
||||
key_names = @keys.collect(&:to_s)
|
||||
target_columns = @table.columns.reject do |column|
|
||||
key_names.include?(column.name)
|
||||
end
|
||||
aggregate(target_columns) do |column, indexes|
|
||||
n = 0
|
||||
indexes.each do |index|
|
||||
n += 1 unless column.null?(index)
|
||||
end
|
||||
n
|
||||
end
|
||||
end
|
||||
|
||||
def sum
|
||||
key_names = @keys.collect(&:to_s)
|
||||
target_columns = @table.columns.reject do |column|
|
||||
key_names.include?(column.name) or
|
||||
not column.data_type.is_a?(NumericDataType)
|
||||
end
|
||||
aggregate(target_columns) do |column, indexes|
|
||||
n = 0
|
||||
indexes.each do |index|
|
||||
value = column[index]
|
||||
n += value unless value.nil?
|
||||
end
|
||||
n
|
||||
end
|
||||
end
|
||||
|
||||
def average
|
||||
key_names = @keys.collect(&:to_s)
|
||||
target_columns = @table.columns.reject do |column|
|
||||
key_names.include?(column.name) or
|
||||
not column.data_type.is_a?(NumericDataType)
|
||||
end
|
||||
aggregate(target_columns) do |column, indexes|
|
||||
average = 0.0
|
||||
n = 0
|
||||
indexes.each do |index|
|
||||
value = column[index]
|
||||
unless value.nil?
|
||||
n += 1
|
||||
average += (value - average) / n
|
||||
end
|
||||
end
|
||||
average
|
||||
end
|
||||
end
|
||||
|
||||
def min
|
||||
key_names = @keys.collect(&:to_s)
|
||||
target_columns = @table.columns.reject do |column|
|
||||
key_names.include?(column.name) or
|
||||
not column.data_type.is_a?(NumericDataType)
|
||||
end
|
||||
aggregate(target_columns) do |column, indexes|
|
||||
n = nil
|
||||
indexes.each do |index|
|
||||
value = column[index]
|
||||
next if value.nil?
|
||||
n ||= value
|
||||
n = value if value < n
|
||||
end
|
||||
n
|
||||
end
|
||||
end
|
||||
|
||||
def max
|
||||
key_names = @keys.collect(&:to_s)
|
||||
target_columns = @table.columns.reject do |column|
|
||||
key_names.include?(column.name) or
|
||||
not column.data_type.is_a?(NumericDataType)
|
||||
end
|
||||
aggregate(target_columns) do |column, indexes|
|
||||
n = nil
|
||||
indexes.each do |index|
|
||||
value = column[index]
|
||||
next if value.nil?
|
||||
n ||= value
|
||||
n = value if value > n
|
||||
end
|
||||
n
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def aggregate(target_columns)
|
||||
sort_values = @table.n_rows.times.collect do |i|
|
||||
key_values = @keys.collect do |key|
|
||||
@table[key][i]
|
||||
end
|
||||
[key_values, i]
|
||||
end
|
||||
sorted = sort_values.sort_by do |key_values, i|
|
||||
key_values
|
||||
end
|
||||
|
||||
grouped_keys = []
|
||||
aggregated_arrays_raw = []
|
||||
target_columns.size.times do
|
||||
aggregated_arrays_raw << []
|
||||
end
|
||||
indexes = []
|
||||
sorted.each do |key_values, i|
|
||||
if grouped_keys.empty?
|
||||
grouped_keys << key_values
|
||||
indexes.clear
|
||||
indexes << i
|
||||
else
|
||||
if key_values == grouped_keys.last
|
||||
indexes << i
|
||||
else
|
||||
grouped_keys << key_values
|
||||
target_columns.each_with_index do |column, j|
|
||||
aggregated_arrays_raw[j] << yield(column, indexes)
|
||||
end
|
||||
indexes.clear
|
||||
indexes << i
|
||||
end
|
||||
end
|
||||
end
|
||||
target_columns.each_with_index do |column, j|
|
||||
aggregated_arrays_raw[j] << yield(column, indexes)
|
||||
end
|
||||
|
||||
grouped_key_arrays_raw = grouped_keys.transpose
|
||||
columns = @keys.collect.with_index do |key, i|
|
||||
key_column = @table[key]
|
||||
key_column_array_class = key_column.data.chunks.first.class
|
||||
if key_column_array_class == TimestampArray
|
||||
builder = TimestampArrayBuilder.new(key_column.data_type)
|
||||
key_column_array = builder.build(grouped_key_arrays_raw[i])
|
||||
else
|
||||
key_column_array =
|
||||
key_column_array_class.new(grouped_key_arrays_raw[i])
|
||||
end
|
||||
Column.new(key_column.field, key_column_array)
|
||||
end
|
||||
target_columns.each_with_index do |column, i|
|
||||
array = ArrayBuilder.build(aggregated_arrays_raw[i])
|
||||
field = Field.new(column.name, array.value_data_type)
|
||||
columns << Column.new(field, array)
|
||||
end
|
||||
Table.new(columns)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,93 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "arrow/block-closable"
|
||||
|
||||
module Arrow
|
||||
class Loader < GObjectIntrospection::Loader
|
||||
class << self
|
||||
def load
|
||||
super("Arrow", Arrow)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def post_load(repository, namespace)
|
||||
require_libraries
|
||||
end
|
||||
|
||||
def require_libraries
|
||||
require "arrow/array"
|
||||
require "arrow/array-builder"
|
||||
require "arrow/chunked-array"
|
||||
require "arrow/column"
|
||||
require "arrow/csv-loader"
|
||||
require "arrow/csv-reader"
|
||||
require "arrow/date32-array"
|
||||
require "arrow/date32-array-builder"
|
||||
require "arrow/date64-array"
|
||||
require "arrow/date64-array-builder"
|
||||
require "arrow/field"
|
||||
require "arrow/record"
|
||||
require "arrow/record-batch"
|
||||
require "arrow/rolling-window"
|
||||
require "arrow/slicer"
|
||||
require "arrow/table"
|
||||
require "arrow/table-formatter"
|
||||
require "arrow/table-list-formatter"
|
||||
require "arrow/table-table-formatter"
|
||||
require "arrow/table-loader"
|
||||
require "arrow/table-saver"
|
||||
require "arrow/tensor"
|
||||
require "arrow/timestamp-array"
|
||||
require "arrow/timestamp-array-builder"
|
||||
|
||||
require "arrow/record-batch-file-reader"
|
||||
require "arrow/record-batch-stream-reader"
|
||||
end
|
||||
|
||||
def load_object_info(info)
|
||||
super
|
||||
|
||||
klass = @base_module.const_get(rubyish_class_name(info))
|
||||
if klass.method_defined?(:close)
|
||||
klass.extend(BlockClosable)
|
||||
end
|
||||
end
|
||||
|
||||
def load_method_info(info, klass, method_name)
|
||||
case klass.name
|
||||
when "Arrow::StringArray"
|
||||
case method_name
|
||||
when "get_value"
|
||||
method_name = "get_raw_value"
|
||||
when "get_string"
|
||||
method_name = "get_value"
|
||||
end
|
||||
super(info, klass, method_name)
|
||||
when "Arrow::TimestampArray", "Arrow::Date32Array", "Arrow::Date64Array"
|
||||
case method_name
|
||||
when "get_value"
|
||||
method_name = "get_raw_value"
|
||||
end
|
||||
super(info, klass, method_name)
|
||||
else
|
||||
super
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,28 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class RecordBatchFileReader
|
||||
include Enumerable
|
||||
|
||||
def each
|
||||
n_record_batches.times do |i|
|
||||
yield(get_record_batch(i))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,30 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class RecordBatchStreamReader
|
||||
include Enumerable
|
||||
|
||||
def each
|
||||
loop do
|
||||
record_batch = next_record_batch
|
||||
break if record_batch.nil?
|
||||
yield(record_batch)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,45 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "arrow/record-containable"
|
||||
|
||||
module Arrow
|
||||
class RecordBatch
|
||||
include RecordContainable
|
||||
include Enumerable
|
||||
|
||||
alias_method :each, :each_record
|
||||
|
||||
alias_method :columns_raw, :columns
|
||||
def columns
|
||||
@columns ||= columns_raw
|
||||
end
|
||||
|
||||
def respond_to_missing?(name, include_private)
|
||||
return true if find_column(name)
|
||||
super
|
||||
end
|
||||
|
||||
def method_missing(name, *args, &block)
|
||||
if args.empty?
|
||||
column = find_column(name)
|
||||
return column if column
|
||||
end
|
||||
super
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,73 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
module RecordContainable
|
||||
def each_column(&block)
|
||||
return to_enum(__method__) unless block_given?
|
||||
|
||||
columns.each(&block)
|
||||
end
|
||||
|
||||
def each_record(reuse_record: false)
|
||||
unless block_given?
|
||||
return to_enum(__method__, reuse_record: reuse_record)
|
||||
end
|
||||
|
||||
if reuse_record
|
||||
record = Record.new(self, nil)
|
||||
n_rows.times do |i|
|
||||
record.index = i
|
||||
yield(record)
|
||||
end
|
||||
else
|
||||
n_rows.times do |i|
|
||||
yield(Record.new(self, i))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def find_column(name_or_index)
|
||||
case name_or_index
|
||||
when String, Symbol
|
||||
name = name_or_index.to_s
|
||||
index = resolve_column_name(name)
|
||||
return nil if index.nil?
|
||||
columns[index]
|
||||
when Integer
|
||||
index = name_or_index
|
||||
columns[index]
|
||||
else
|
||||
message = "column name or index must be String, Symbol or Integer"
|
||||
raise ArgumentError, message
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def resolve_column_name(name)
|
||||
(@column_name_to_index ||= build_column_name_resolve_table)[name]
|
||||
end
|
||||
|
||||
def build_column_name_resolve_table
|
||||
table = {}
|
||||
schema.fields.each_with_index do |field, i|
|
||||
table[field.name] = i
|
||||
end
|
||||
table
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,57 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class Record
|
||||
attr_accessor :index
|
||||
def initialize(record_container, index)
|
||||
@record_container = record_container
|
||||
@index = index
|
||||
end
|
||||
|
||||
def [](column_name_or_column_index)
|
||||
column = @record_container.find_column(column_name_or_column_index)
|
||||
return nil if column.nil?
|
||||
column[@index]
|
||||
end
|
||||
|
||||
def columns
|
||||
@record_container.columns
|
||||
end
|
||||
|
||||
def to_h
|
||||
attributes = {}
|
||||
@record_container.schema.fields.each_with_index do |field, i|
|
||||
attributes[field.name] = self[i]
|
||||
end
|
||||
attributes
|
||||
end
|
||||
|
||||
def respond_to_missing?(name, include_private)
|
||||
return true if @record_container.find_column(name)
|
||||
super
|
||||
end
|
||||
|
||||
def method_missing(name, *args, &block)
|
||||
if args.empty?
|
||||
column = @record_container.find_column(name)
|
||||
return column[@index] if column
|
||||
end
|
||||
super
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,48 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
# Experimental
|
||||
#
|
||||
# TODO: Almost codes should be implemented in Apache Arrow C++.
|
||||
class RollingWindow
|
||||
def initialize(table, size)
|
||||
@table = table
|
||||
@size = size
|
||||
end
|
||||
|
||||
def lag(key, diff: 1)
|
||||
column = @table[key]
|
||||
if @size
|
||||
windows = column.each_slice(@size)
|
||||
else
|
||||
windows = column
|
||||
end
|
||||
lag_values = [nil] * diff
|
||||
windows.each_cons(diff + 1) do |values|
|
||||
target = values[0]
|
||||
current = values[1]
|
||||
if target.nil? or current.nil?
|
||||
lag_values << nil
|
||||
else
|
||||
lag_values << current - target
|
||||
end
|
||||
end
|
||||
ArrayBuilder.build(lag_values)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,454 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
# Experimental
|
||||
#
|
||||
# TODO: Almost codes should be implemented in Apache Arrow C++.
|
||||
class Slicer
|
||||
def initialize(table)
|
||||
@table = table
|
||||
end
|
||||
|
||||
def [](column_name)
|
||||
column = @table[column_name]
|
||||
return nil if column.nil?
|
||||
ColumnCondition.new(column)
|
||||
end
|
||||
|
||||
def respond_to_missing?(name, include_private)
|
||||
return true if self[name]
|
||||
super
|
||||
end
|
||||
|
||||
def method_missing(name, *args, &block)
|
||||
if args.empty?
|
||||
column_condition = self[name]
|
||||
return column_condition if column_condition
|
||||
end
|
||||
super
|
||||
end
|
||||
|
||||
class Condition
|
||||
def evaluate
|
||||
message = "Slicer::Condition must define \#evaluate: #{inspect}"
|
||||
raise NotImplementedError.new(message)
|
||||
end
|
||||
|
||||
def &(condition)
|
||||
AndCondition.new(self, condition)
|
||||
end
|
||||
|
||||
def |(condition)
|
||||
OrCondition.new(self, condition)
|
||||
end
|
||||
|
||||
def ^(condition)
|
||||
XorCondition.new(self, condition)
|
||||
end
|
||||
end
|
||||
|
||||
class LogicalCondition < Condition
|
||||
def initialize(condition1, condition2)
|
||||
@condition1 = condition1
|
||||
@condition2 = condition2
|
||||
end
|
||||
|
||||
def evaluate
|
||||
values1 = @condition1.evaluate.each
|
||||
values2 = @condition2.evaluate.each
|
||||
raw_array = []
|
||||
begin
|
||||
loop do
|
||||
value1 = values1.next
|
||||
value2 = values2.next
|
||||
if value1.nil? or value2.nil?
|
||||
raw_array << nil
|
||||
else
|
||||
raw_array << evaluate_value(value1, value2)
|
||||
end
|
||||
end
|
||||
rescue StopIteration
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
|
||||
class AndCondition < LogicalCondition
|
||||
private
|
||||
def evaluate_value(value1, value2)
|
||||
value1 and value2
|
||||
end
|
||||
end
|
||||
|
||||
class OrCondition < LogicalCondition
|
||||
private
|
||||
def evaluate_value(value1, value2)
|
||||
value1 or value2
|
||||
end
|
||||
end
|
||||
|
||||
class XorCondition < LogicalCondition
|
||||
private
|
||||
def evaluate_value(value1, value2)
|
||||
value1 ^ value2
|
||||
end
|
||||
end
|
||||
|
||||
class ColumnCondition < Condition
|
||||
def initialize(column)
|
||||
@column = column
|
||||
end
|
||||
|
||||
def evaluate
|
||||
data = @column.data
|
||||
|
||||
case @column.data_type
|
||||
when BooleanDataType
|
||||
data
|
||||
else
|
||||
if data.n_chunks == 1
|
||||
data.get_chunk(0).cast(BooleanDataType.new, nil)
|
||||
else
|
||||
arrays = data.each_chunk.collect do |chunk|
|
||||
chunk.cast(BooleanDataType.new, nil)
|
||||
end
|
||||
ChunkedArray.new(arrays)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def !@
|
||||
NotColumnCondition.new(@column)
|
||||
end
|
||||
|
||||
def null?
|
||||
self == nil
|
||||
end
|
||||
|
||||
def valid?
|
||||
self != nil
|
||||
end
|
||||
|
||||
def ==(value)
|
||||
EqualCondition.new(@column, value)
|
||||
end
|
||||
|
||||
def !=(value)
|
||||
NotEqualCondition.new(@column, value)
|
||||
end
|
||||
|
||||
def <(value)
|
||||
LessCondition.new(@column, value)
|
||||
end
|
||||
|
||||
def <=(value)
|
||||
LessEqualCondition.new(@column, value)
|
||||
end
|
||||
|
||||
def >(value)
|
||||
GreaterCondition.new(@column, value)
|
||||
end
|
||||
|
||||
def >=(value)
|
||||
GreaterEqualCondition.new(@column, value)
|
||||
end
|
||||
|
||||
def in?(values)
|
||||
InCondition.new(@column, values)
|
||||
end
|
||||
|
||||
def select(&block)
|
||||
SelectCondition.new(@column, block)
|
||||
end
|
||||
|
||||
def reject(&block)
|
||||
RejectCondition.new(@column, block)
|
||||
end
|
||||
end
|
||||
|
||||
class NotColumnCondition < Condition
|
||||
def initialize(column)
|
||||
@column = column
|
||||
end
|
||||
|
||||
def evaluate
|
||||
data = @column.data
|
||||
raw_array = []
|
||||
data.each_chunk do |chunk|
|
||||
if chunk.is_a?(BooleanArray)
|
||||
boolean_array = chunk
|
||||
else
|
||||
boolean_array = chunk.cast(BooleanDataType.new, nil)
|
||||
end
|
||||
boolean_array.each do |value|
|
||||
if value.nil?
|
||||
raw_array << value
|
||||
else
|
||||
raw_array << !value
|
||||
end
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
|
||||
def !@
|
||||
ColumnCondition.new(@column)
|
||||
end
|
||||
end
|
||||
|
||||
class EqualCondition < Condition
|
||||
def initialize(column, value)
|
||||
@column = column
|
||||
@value = value
|
||||
end
|
||||
|
||||
def !@
|
||||
NotEqualCondition.new(@column, @value)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
case @value
|
||||
when nil
|
||||
raw_array = @column.collect(&:nil?)
|
||||
BooleanArray.new(raw_array)
|
||||
else
|
||||
raw_array = @column.collect do |value|
|
||||
if value.nil?
|
||||
nil
|
||||
else
|
||||
@value == value
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class NotEqualCondition < Condition
|
||||
def initialize(column, value)
|
||||
@column = column
|
||||
@value = value
|
||||
end
|
||||
|
||||
def !@
|
||||
EqualCondition.new(@column, @value)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
case @value
|
||||
when nil
|
||||
if @column.n_nulls.zero?
|
||||
raw_array = [true] * @column.length
|
||||
else
|
||||
raw_array = @column.length.times.collect do |i|
|
||||
@column.valid?(i)
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
else
|
||||
raw_array = @column.collect do |value|
|
||||
if value.nil?
|
||||
nil
|
||||
else
|
||||
@value != value
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class LessCondition < Condition
|
||||
def initialize(column, value)
|
||||
@column = column
|
||||
@value = value
|
||||
end
|
||||
|
||||
def !@
|
||||
GreaterEqualCondition.new(@column, @value)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
raw_array = @column.collect do |value|
|
||||
if value.nil?
|
||||
nil
|
||||
else
|
||||
@value > value
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
|
||||
class LessEqualCondition < Condition
|
||||
def initialize(column, value)
|
||||
@column = column
|
||||
@value = value
|
||||
end
|
||||
|
||||
def !@
|
||||
GreaterCondition.new(@column, @value)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
raw_array = @column.collect do |value|
|
||||
if value.nil?
|
||||
nil
|
||||
else
|
||||
@value >= value
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
|
||||
class GreaterCondition < Condition
|
||||
def initialize(column, value)
|
||||
@column = column
|
||||
@value = value
|
||||
end
|
||||
|
||||
def !@
|
||||
LessEqualCondition.new(@column, @value)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
raw_array = @column.collect do |value|
|
||||
if value.nil?
|
||||
nil
|
||||
else
|
||||
@value < value
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
|
||||
class GreaterEqualCondition < Condition
|
||||
def initialize(column, value)
|
||||
@column = column
|
||||
@value = value
|
||||
end
|
||||
|
||||
def !@
|
||||
LessCondition.new(@column, @value)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
raw_array = @column.collect do |value|
|
||||
if value.nil?
|
||||
nil
|
||||
else
|
||||
@value <= value
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
|
||||
class InCondition < Condition
|
||||
def initialize(column, values)
|
||||
@column = column
|
||||
@values = values
|
||||
end
|
||||
|
||||
def !@
|
||||
NotInCondition.new(@column, @values)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
values_index = {}
|
||||
@values.each do |value|
|
||||
values_index[value] = true
|
||||
end
|
||||
raw_array = @column.collect do |value|
|
||||
if value.nil?
|
||||
nil
|
||||
else
|
||||
values_index.key?(value)
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
|
||||
class NotInCondition < Condition
|
||||
def initialize(column, values)
|
||||
@column = column
|
||||
@values = values
|
||||
end
|
||||
|
||||
def !@
|
||||
InCondition.new(@column, @values)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
values_index = {}
|
||||
@values.each do |value|
|
||||
values_index[value] = true
|
||||
end
|
||||
raw_array = @column.collect do |value|
|
||||
if value.nil?
|
||||
nil
|
||||
else
|
||||
not values_index.key?(value)
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
|
||||
class SelectCondition < Condition
|
||||
def initialize(column, block)
|
||||
@column = column
|
||||
@block = block
|
||||
end
|
||||
|
||||
def !@
|
||||
RejectCondition.new(@column, @block)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
BooleanArray.new(@column.collect(&@block))
|
||||
end
|
||||
end
|
||||
|
||||
class RejectCondition < Condition
|
||||
def initialize(column, block)
|
||||
@column = column
|
||||
@block = block
|
||||
end
|
||||
|
||||
def !@
|
||||
SelectCondition.new(@column, @block)
|
||||
end
|
||||
|
||||
def evaluate
|
||||
raw_array = @column.collect do |value|
|
||||
evaluated_value = @block.call(value)
|
||||
if evaluated_value.nil?
|
||||
nil
|
||||
else
|
||||
not evaluated_value
|
||||
end
|
||||
end
|
||||
BooleanArray.new(raw_array)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,66 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
# TODO: Almost codes should be implemented in Apache Arrow C++.
|
||||
class TableFormatter
|
||||
def initialize(table, options={})
|
||||
@table = table
|
||||
@options = options
|
||||
end
|
||||
|
||||
def format
|
||||
text = ""
|
||||
columns = @table.columns
|
||||
format_header(text, columns)
|
||||
|
||||
n_rows = @table.n_rows
|
||||
return text if n_rows.zero?
|
||||
|
||||
border = @options[:border] || 10
|
||||
n_digits = (Math.log10(n_rows) + 1).truncate
|
||||
head_limit = [border, n_rows].min
|
||||
head_column_values = columns.collect do |column|
|
||||
column.each.take(head_limit)
|
||||
end
|
||||
format_rows(text,
|
||||
columns,
|
||||
head_column_values.transpose,
|
||||
n_digits,
|
||||
0)
|
||||
return text if n_rows <= border
|
||||
|
||||
tail_start = [border, n_rows - border].max
|
||||
tail_limit = n_rows - tail_start
|
||||
tail_column_values = columns.collect do |column|
|
||||
column.reverse_each.take(tail_limit).reverse
|
||||
end
|
||||
|
||||
if head_limit != tail_start
|
||||
format_ellipsis(text)
|
||||
end
|
||||
|
||||
format_rows(text,
|
||||
columns,
|
||||
tail_column_values.transpose,
|
||||
n_digits,
|
||||
tail_start)
|
||||
|
||||
text
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,39 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
# TODO: Almost codes should be implemented in Apache Arrow C++.
|
||||
class TableListFormatter < TableFormatter
|
||||
private
|
||||
def format_header(text, columns)
|
||||
end
|
||||
|
||||
def format_rows(text, columns, rows, n_digits, start_offset)
|
||||
rows.each_with_index do |row, nth_row|
|
||||
text << ("=" * 20 + " #{start_offset + nth_row} " + "=" * 20 + "\n")
|
||||
row.each_with_index do |column_value, nth_column|
|
||||
column = columns[nth_column]
|
||||
text << "#{column.name}: #{column_value}\n"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def format_ellipsis(text)
|
||||
text << "...\n"
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,120 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class TableLoader
|
||||
class << self
|
||||
def load(path, options={})
|
||||
new(path, options).load
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(path, options={})
|
||||
@path = path
|
||||
@options = options
|
||||
end
|
||||
|
||||
def load
|
||||
path = @path
|
||||
path = path.to_path if path.respond_to?(:to_path)
|
||||
format = @options[:format] || guess_format(path) || :arrow
|
||||
|
||||
custom_load_method = "load_as_#{format}"
|
||||
unless respond_to?(custom_load_method, true)
|
||||
available_formats = []
|
||||
(methods(true) | private_methods(true)).each do |name|
|
||||
match_data = /\Aload_as_/.match(name.to_s)
|
||||
if match_data
|
||||
available_formats << match_data.post_match
|
||||
end
|
||||
end
|
||||
message = "Arrow::Table load format must be one of ["
|
||||
message << available_formats.join(", ")
|
||||
message << "]: #{format.inspect}"
|
||||
raise ArgumentError, message
|
||||
end
|
||||
__send__(custom_load_method, path)
|
||||
end
|
||||
|
||||
private
|
||||
def guess_format(path)
|
||||
extension = ::File.extname(path).gsub(/\A\./, "").downcase
|
||||
return nil if extension.empty?
|
||||
|
||||
return extension if respond_to?("load_as_#{extension}", true)
|
||||
|
||||
nil
|
||||
end
|
||||
|
||||
def load_raw(input, reader)
|
||||
schema = reader.schema
|
||||
chunked_arrays = []
|
||||
reader.each do |record_batch|
|
||||
record_batch.columns.each_with_index do |array, i|
|
||||
chunked_array = (chunked_arrays[i] ||= [])
|
||||
chunked_array << array
|
||||
end
|
||||
end
|
||||
columns = schema.fields.collect.with_index do |field, i|
|
||||
Column.new(field, ChunkedArray.new(chunked_arrays[i]))
|
||||
end
|
||||
table = Table.new(schema, columns)
|
||||
table.instance_variable_set(:@input, input)
|
||||
table
|
||||
end
|
||||
|
||||
def load_as_arrow(path)
|
||||
input = nil
|
||||
reader = nil
|
||||
error = nil
|
||||
reader_class_candidates = [
|
||||
RecordBatchFileReader,
|
||||
RecordBatchStreamReader,
|
||||
]
|
||||
reader_class_candidates.each do |reader_class_candidate|
|
||||
input = MemoryMappedInputStream.new(path)
|
||||
begin
|
||||
reader = reader_class_candidate.new(input)
|
||||
rescue Arrow::Error
|
||||
error = $!
|
||||
else
|
||||
break
|
||||
end
|
||||
end
|
||||
raise error if reader.nil?
|
||||
load_raw(input, reader)
|
||||
end
|
||||
|
||||
def load_as_batch(path)
|
||||
input = MemoryMappedInputStream.new(path)
|
||||
reader = RecordBatchFileReader.new(input)
|
||||
load_raw(input, reader)
|
||||
end
|
||||
|
||||
def load_as_stream(path)
|
||||
input = MemoryMappedInputStream.new(path)
|
||||
reader = RecordBatchStreamReader.new(input)
|
||||
load_raw(input, reader)
|
||||
end
|
||||
|
||||
def load_as_csv(path)
|
||||
options = @options.dup
|
||||
options.delete(:format)
|
||||
CSVLoader.load(Pathname.new(path), options)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,96 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class TableSaver
|
||||
class << self
|
||||
def save(table, path, options={})
|
||||
new(table, path, options).save
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(table, path, options={})
|
||||
@table = table
|
||||
@path = path
|
||||
@options = options
|
||||
end
|
||||
|
||||
def save
|
||||
path = @path
|
||||
path = path.to_path if path.respond_to?(:to_path)
|
||||
format = @options[:format] || guess_format(path) || :arrow
|
||||
|
||||
custom_save_method = "save_as_#{format}"
|
||||
unless respond_to?(custom_save_method, true)
|
||||
available_formats = []
|
||||
(methods(true) | private_methods(true)).each do |name|
|
||||
match_data = /\Asave_as_/.match(name.to_s)
|
||||
if match_data
|
||||
available_formats << match_data.post_match
|
||||
end
|
||||
end
|
||||
message = "Arrow::Table save format must be one of ["
|
||||
message << available_formats.join(", ")
|
||||
message << "]: #{format.inspect}"
|
||||
raise ArgumentError, message
|
||||
end
|
||||
__send__(custom_save_method, path)
|
||||
end
|
||||
|
||||
private
|
||||
def guess_format(path)
|
||||
extension = ::File.extname(path).gsub(/\A\./, "").downcase
|
||||
return nil if extension.empty?
|
||||
|
||||
return extension if respond_to?("save_as_#{extension}", true)
|
||||
|
||||
nil
|
||||
end
|
||||
|
||||
def save_raw(writer_class, path)
|
||||
FileOutputStream.open(path, false) do |output|
|
||||
writer_class.open(output, @table.schema) do |writer|
|
||||
writer.write_table(@table)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def save_as_arrow(path)
|
||||
save_as_batch(path)
|
||||
end
|
||||
|
||||
def save_as_batch(path)
|
||||
save_raw(RecordBatchFileWriter, path)
|
||||
end
|
||||
|
||||
def save_as_stream(path)
|
||||
save_raw(RecordBatchStreamWriter, path)
|
||||
end
|
||||
|
||||
def save_as_csv(path)
|
||||
CSV.open(path, "w") do |csv|
|
||||
names = @table.schema.fields.collect(&:name)
|
||||
csv << names
|
||||
@table.each_record(reuse_record: true) do |record|
|
||||
csv << names.collect do |name|
|
||||
record[name]
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,73 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "time"
|
||||
|
||||
module Arrow
|
||||
# TODO: Almost codes should be implemented in Apache Arrow C++.
|
||||
class TableTableFormatter < TableFormatter
|
||||
private
|
||||
def format_header(text, columns)
|
||||
columns.each do |column|
|
||||
text << "\t"
|
||||
text << format_column_name(column)
|
||||
end
|
||||
text << "\n"
|
||||
end
|
||||
|
||||
FLOAT_N_DIGITS = 10
|
||||
def format_column_name(column)
|
||||
case column.data_type
|
||||
when TimestampDataType
|
||||
"%*s" % [Time.now.iso8601.size, column.name]
|
||||
when FloatDataType, DoubleDataType
|
||||
"%*s" % [FLOAT_N_DIGITS, column.name]
|
||||
else
|
||||
column.name
|
||||
end
|
||||
end
|
||||
|
||||
def format_rows(text, columns, rows, n_digits, start_offset)
|
||||
rows.each_with_index do |row, nth_row|
|
||||
text << ("%*d" % [n_digits, start_offset + nth_row])
|
||||
row.each_with_index do |column_value, nth_column|
|
||||
text << "\t"
|
||||
column = columns[nth_column]
|
||||
text << format_column_value(column, column_value)
|
||||
end
|
||||
text << "\n"
|
||||
end
|
||||
end
|
||||
|
||||
def format_column_value(column, value)
|
||||
case value
|
||||
when Time
|
||||
value.iso8601
|
||||
when Float
|
||||
"%*f" % [[column.name.size, FLOAT_N_DIGITS].max, value]
|
||||
when Integer
|
||||
"%*d" % [column.name.size, value]
|
||||
else
|
||||
"%-*s" % [column.name.size, value.to_s]
|
||||
end
|
||||
end
|
||||
|
||||
def format_ellipsis(text)
|
||||
text << "...\n"
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,381 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "arrow/group"
|
||||
require "arrow/record-containable"
|
||||
|
||||
module Arrow
|
||||
class Table
|
||||
include RecordContainable
|
||||
|
||||
class << self
|
||||
def load(path, options={})
|
||||
TableLoader.load(path, options)
|
||||
end
|
||||
end
|
||||
|
||||
alias_method :initialize_raw, :initialize
|
||||
def initialize(schema_or_raw_table_or_columns, columns=nil)
|
||||
if columns.nil?
|
||||
if schema_or_raw_table_or_columns[0].is_a?(Column)
|
||||
columns = schema_or_raw_table_or_columns
|
||||
fields = columns.collect(&:field)
|
||||
schema = Schema.new(fields)
|
||||
else
|
||||
raw_table = schema_or_raw_table_or_columns
|
||||
fields = []
|
||||
columns = []
|
||||
raw_table.each do |name, array|
|
||||
field = Field.new(name.to_s, array.value_data_type)
|
||||
fields << field
|
||||
columns << Column.new(field, array)
|
||||
end
|
||||
schema = Schema.new(fields)
|
||||
end
|
||||
else
|
||||
schema = schema_or_raw_table_or_columns
|
||||
end
|
||||
initialize_raw(schema, columns)
|
||||
end
|
||||
|
||||
def columns
|
||||
@columns ||= n_columns.times.collect {|i| get_column(i)}
|
||||
end
|
||||
|
||||
def each_record_batch
|
||||
return to_enum(__method__) unless block_given?
|
||||
|
||||
reader = TableBatchReader.new(self)
|
||||
while record_batch = reader.read_next
|
||||
yield(record_batch)
|
||||
end
|
||||
end
|
||||
|
||||
alias_method :size, :n_rows
|
||||
alias_method :length, :n_rows
|
||||
|
||||
alias_method :[], :find_column
|
||||
|
||||
# TODO
|
||||
#
|
||||
# @return [Arrow::Table]
|
||||
def slice(*args)
|
||||
slicers = []
|
||||
expected_n_args = nil
|
||||
case args.size
|
||||
when 0
|
||||
expected_n_args = "1..2" unless block_given?
|
||||
when 1
|
||||
slicers << args[0]
|
||||
when 2
|
||||
from, to = args
|
||||
slicers << (from...(from + to))
|
||||
else
|
||||
if block_given?
|
||||
expected_n_args = "0..2"
|
||||
else
|
||||
expected_n_args = "1..2"
|
||||
end
|
||||
end
|
||||
if expected_n_args
|
||||
message = "wrong number of arguments " +
|
||||
"(given #{args.size}, expected #{expected_n_args})"
|
||||
raise ArgumentError, message
|
||||
end
|
||||
|
||||
if block_given?
|
||||
block_slicer = yield(Slicer.new(self))
|
||||
case block_slicer
|
||||
when nil
|
||||
# Ignore
|
||||
when ::Array
|
||||
slicers.concat(block_slicer)
|
||||
else
|
||||
slicers << block_slicer
|
||||
end
|
||||
end
|
||||
|
||||
ranges = []
|
||||
slicers.each do |slicer|
|
||||
slicer = slicer.evaluate if slicer.respond_to?(:evaluate)
|
||||
case slicer
|
||||
when Integer
|
||||
slicer += n_rows if slicer < 0
|
||||
ranges << [slicer, slicer]
|
||||
when Range
|
||||
from = slicer.first
|
||||
to = slicer.last
|
||||
to -= 1 if slicer.exclude_end?
|
||||
from += n_rows if from < 0
|
||||
to += n_rows if to < 0
|
||||
ranges << [from, to]
|
||||
when ::Array
|
||||
boolean_array_to_slice_ranges(slicer, 0, ranges)
|
||||
when ChunkedArray
|
||||
offset = 0
|
||||
slicer.each_chunk do |array|
|
||||
boolean_array_to_slice_ranges(array, offset, ranges)
|
||||
offset += array.length
|
||||
end
|
||||
when BooleanArray
|
||||
boolean_array_to_slice_ranges(slicer, 0, ranges)
|
||||
else
|
||||
message = "slicer must be Integer, Range, (from, to), " +
|
||||
"Arrow::ChunkedArray of Arrow::BooleanArray, " +
|
||||
"Arrow::BooleanArray or Arrow::Slicer::Condition: #{slicer.inspect}"
|
||||
raise ArgumentError, message
|
||||
end
|
||||
end
|
||||
slice_by_ranges(ranges)
|
||||
end
|
||||
|
||||
# TODO
|
||||
#
|
||||
# @return [Arrow::Table]
|
||||
def merge(other)
|
||||
added_columns = {}
|
||||
removed_columns = {}
|
||||
|
||||
case other
|
||||
when Hash
|
||||
other.each do |name, value|
|
||||
name = name.to_s
|
||||
if value
|
||||
added_columns[name] = ensure_column(name, value)
|
||||
else
|
||||
removed_columns[name] = true
|
||||
end
|
||||
end
|
||||
when Table
|
||||
added_columns = {}
|
||||
other.columns.each do |column|
|
||||
added_columns[column.name] = column
|
||||
end
|
||||
else
|
||||
message = "merge target must be Hash or Arrow::Table: " +
|
||||
"<#{other.inspect}>: #{inspect}"
|
||||
raise ArgumentError, message
|
||||
end
|
||||
|
||||
new_columns = []
|
||||
columns.each do |column|
|
||||
column_name = column.name
|
||||
new_column = added_columns.delete(column_name)
|
||||
if new_column
|
||||
new_columns << new_column
|
||||
next
|
||||
end
|
||||
next if removed_columns.key?(column_name)
|
||||
new_columns << column
|
||||
end
|
||||
added_columns.each do |name, new_column|
|
||||
new_columns << new_column
|
||||
end
|
||||
new_fields = new_columns.collect do |new_column|
|
||||
new_column.field
|
||||
end
|
||||
self.class.new(Schema.new(new_fields), new_columns)
|
||||
end
|
||||
|
||||
alias_method :remove_column_raw, :remove_column
|
||||
def remove_column(name_or_index)
|
||||
case name_or_index
|
||||
when String, Symbol
|
||||
name = name_or_index.to_s
|
||||
index = columns.index {|column| column.name == name}
|
||||
if index.nil?
|
||||
message = "unknown column: #{name_or_index.inspect}: #{inspect}"
|
||||
raise KeyError.new(message)
|
||||
end
|
||||
else
|
||||
index = name_or_index
|
||||
index += n_columns if index < 0
|
||||
if index < 0 or index >= n_columns
|
||||
message = "out of index (0..#{n_columns - 1}): " +
|
||||
"#{name_or_index.inspect}: #{inspect}"
|
||||
raise IndexError.new(message)
|
||||
end
|
||||
end
|
||||
remove_column_raw(index)
|
||||
end
|
||||
|
||||
# TODO
|
||||
#
|
||||
# @return [Arrow::Table]
|
||||
def select_columns(*selectors, &block)
|
||||
if selectors.empty?
|
||||
return to_enum(__method__) unless block_given?
|
||||
selected_columns = columns.select(&block)
|
||||
else
|
||||
selected_columns = []
|
||||
selectors.each do |selector|
|
||||
case selector
|
||||
when String, Symbol
|
||||
column = find_column(selector)
|
||||
if column.nil?
|
||||
message = "unknown column: #{selector.inspect}: #{inspect}"
|
||||
raise KeyError.new(message)
|
||||
end
|
||||
selected_columns << column
|
||||
when Range
|
||||
selected_columns.concat(columns[selector])
|
||||
else
|
||||
column = columns[selector]
|
||||
if column.nil?
|
||||
message = "out of index (0..#{n_columns - 1}): " +
|
||||
"#{selector.inspect}: #{inspect}"
|
||||
raise IndexError.new(message)
|
||||
end
|
||||
selected_columns << column
|
||||
end
|
||||
end
|
||||
selected_columns = selected_columns.select(&block) if block_given?
|
||||
end
|
||||
self.class.new(selected_columns)
|
||||
end
|
||||
|
||||
# Experimental
|
||||
def group(*keys)
|
||||
Group.new(self, keys)
|
||||
end
|
||||
|
||||
# Experimental
|
||||
def window(size: nil)
|
||||
RollingWindow.new(self, size)
|
||||
end
|
||||
|
||||
def save(path, options={})
|
||||
saver = TableSaver.new(self, path, options)
|
||||
saver.save
|
||||
end
|
||||
|
||||
def pack
|
||||
packed_columns = columns.collect do |column|
|
||||
column.pack
|
||||
end
|
||||
self.class.new(schema, packed_columns)
|
||||
end
|
||||
|
||||
def to_s(options={})
|
||||
case options[:format]
|
||||
when :list
|
||||
formatter_class = TableListFormatter
|
||||
else
|
||||
formatter_class = TableTableFormatter
|
||||
end
|
||||
formatter = formatter_class.new(self, options)
|
||||
formatter.format
|
||||
end
|
||||
|
||||
def inspect
|
||||
"#{super}\n#{to_s}"
|
||||
end
|
||||
|
||||
def respond_to_missing?(name, include_private)
|
||||
return true if find_column(name)
|
||||
super
|
||||
end
|
||||
|
||||
def method_missing(name, *args, &block)
|
||||
if args.empty?
|
||||
column = find_column(name)
|
||||
return column if column
|
||||
end
|
||||
super
|
||||
end
|
||||
|
||||
private
|
||||
def boolean_array_to_slice_ranges(array, offset, ranges)
|
||||
in_target = false
|
||||
target_start = nil
|
||||
array.each_with_index do |is_target, i|
|
||||
if is_target
|
||||
unless in_target
|
||||
target_start = offset + i
|
||||
in_target = true
|
||||
end
|
||||
else
|
||||
if in_target
|
||||
ranges << [target_start, offset + i - 1]
|
||||
target_start = nil
|
||||
in_target = false
|
||||
end
|
||||
end
|
||||
end
|
||||
if in_target
|
||||
ranges << [target_start, offset + array.length - 1]
|
||||
end
|
||||
end
|
||||
|
||||
# TODO: Almost codes should be implemented in Apache Arrow C++.
|
||||
def slice_by_ranges(ranges)
|
||||
sliced_columns = columns.collect do |column|
|
||||
chunks = []
|
||||
arrays = column.data.each_chunk.to_a
|
||||
offset = 0
|
||||
offset_in_array = 0
|
||||
ranges.each do |from, to|
|
||||
range_size = to - from + 1
|
||||
while range_size > 0
|
||||
while offset + arrays.first.length - offset_in_array < from
|
||||
offset += arrays.first.length - offset_in_array
|
||||
arrays.shift
|
||||
offset_in_array = 0
|
||||
end
|
||||
if offset < from
|
||||
skipped_size = from - offset
|
||||
offset += skipped_size
|
||||
offset_in_array += skipped_size
|
||||
end
|
||||
array = arrays.first
|
||||
array_length = array.length
|
||||
rest_length = array_length - offset_in_array
|
||||
if rest_length <= range_size
|
||||
chunks << array.slice(offset_in_array, array_length)
|
||||
offset += rest_length
|
||||
range_size -= rest_length
|
||||
offset_in_array = 0
|
||||
arrays.shift
|
||||
else
|
||||
chunks << array.slice(offset_in_array, range_size)
|
||||
offset += range_size
|
||||
offset_in_array += range_size
|
||||
range_size = 0
|
||||
end
|
||||
end
|
||||
end
|
||||
Column.new(column.field, ChunkedArray.new(chunks))
|
||||
end
|
||||
|
||||
self.class.new(schema, sliced_columns)
|
||||
end
|
||||
|
||||
def ensure_column(name, data)
|
||||
case data
|
||||
when Array
|
||||
field = Field.new(name, data.value_data_type)
|
||||
Column.new(field, data)
|
||||
when Column
|
||||
data
|
||||
else
|
||||
message = "column must be Arrow::Array or Arrow::Column: " +
|
||||
"<#{name}>: <#{data.inspect}>: #{inspect}"
|
||||
raise ArgumentError, message
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,24 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class Tensor
|
||||
def to_arrow
|
||||
self
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,46 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class TimestampArrayBuilder
|
||||
private
|
||||
def unit_id
|
||||
@unit_id ||= value_data_type.unit.nick.to_sym
|
||||
end
|
||||
|
||||
def convert_to_arrow_value(value)
|
||||
if value.respond_to?(:to_time) and not value.is_a?(Time)
|
||||
value = value.to_time
|
||||
end
|
||||
|
||||
if value.is_a?(Time)
|
||||
case unit_id
|
||||
when :second
|
||||
value.to_i
|
||||
when :milli
|
||||
value.to_i * 1_000 + value.usec / 1_000
|
||||
when :micro
|
||||
value.to_i * 1_000_000 + value.usec
|
||||
else
|
||||
value.to_i * 1_000_000_000 + value.nsec
|
||||
end
|
||||
else
|
||||
value
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,54 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
class TimestampArray
|
||||
class << self
|
||||
def new(unit, values)
|
||||
data_type = TimestampDataType.new(unit)
|
||||
builder = TimestampArrayBuilder.new(data_type)
|
||||
builder.build(values)
|
||||
end
|
||||
end
|
||||
|
||||
def get_value(i)
|
||||
to_time(get_raw_value(i))
|
||||
end
|
||||
|
||||
def unit
|
||||
@unit ||= value_data_type.unit
|
||||
end
|
||||
|
||||
private
|
||||
def unit_id
|
||||
@unit_id ||= unit.nick.to_sym
|
||||
end
|
||||
|
||||
def to_time(raw_value)
|
||||
case unit_id
|
||||
when :second
|
||||
Time.at(raw_value)
|
||||
when :milli
|
||||
Time.at(*raw_value.divmod(1_000))
|
||||
when :micro
|
||||
Time.at(*raw_value.divmod(1_000_000))
|
||||
else
|
||||
Time.at(raw_value / 1_000_000_000.0)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,54 @@
|
||||
# -*- ruby -*-
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require_relative "version"
|
||||
|
||||
Gem::Specification.new do |spec|
|
||||
spec.name = "red-arrow"
|
||||
version_components = [
|
||||
Arrow::Version::MAJOR.to_s,
|
||||
Arrow::Version::MINOR.to_s,
|
||||
Arrow::Version::MICRO.to_s,
|
||||
# "beta1",
|
||||
]
|
||||
spec.version = version_components.join(".")
|
||||
spec.homepage = "https://arrow.apache.org/"
|
||||
spec.authors = ["Apache Arrow Developers"]
|
||||
spec.email = ["dev@arrow.apache.org"]
|
||||
|
||||
spec.summary = "Red Arrow is the Ruby bindings of Apache Arrow"
|
||||
spec.description =
|
||||
"Apache Arrow is a common in-memory columnar data store. " +
|
||||
"It's useful to share and process large data."
|
||||
spec.license = "Apache-2.0"
|
||||
spec.files = ["README.md", "Rakefile", "Gemfile", "#{spec.name}.gemspec"]
|
||||
spec.files += Dir.glob("lib/**/*.rb")
|
||||
spec.files += Dir.glob("image/*.*")
|
||||
spec.files += Dir.glob("doc/text/*")
|
||||
spec.test_files += Dir.glob("test/**/*")
|
||||
spec.extensions = ["dependency-check/Rakefile"]
|
||||
|
||||
spec.add_runtime_dependency("gobject-introspection", ">= 3.1.1")
|
||||
spec.add_runtime_dependency("pkg-config")
|
||||
spec.add_runtime_dependency("native-package-installer")
|
||||
|
||||
spec.add_development_dependency("bundler")
|
||||
spec.add_development_dependency("rake")
|
||||
spec.add_development_dependency("test-unit")
|
||||
end
|
||||
@@ -0,0 +1,20 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
score
|
||||
2.9
|
||||
10
|
||||
-1.1
|
||||
|
Can't render this file because it contains an unexpected character in line 6 and column 3.
|
@@ -0,0 +1,20 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
score
|
||||
10
|
||||
2.9
|
||||
-1.1
|
||||
|
Can't render this file because it contains an unexpected character in line 6 and column 3.
|
@@ -0,0 +1,20 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
name,score
|
||||
alice,10
|
||||
bob,""
|
||||
chris,-1
|
||||
|
Can't render this file because it contains an unexpected character in line 6 and column 3.
|
@@ -0,0 +1,20 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
name,score
|
||||
alice,10
|
||||
bob,
|
||||
chris,-1
|
||||
|
Can't render this file because it contains an unexpected character in line 6 and column 3.
|
@@ -0,0 +1,20 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
name,score
|
||||
alice,10.1
|
||||
bob,29.2
|
||||
chris,-1.3
|
||||
|
Can't render this file because it contains an unexpected character in line 6 and column 3.
|
@@ -0,0 +1,20 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
name,score
|
||||
alice,10
|
||||
bob,29
|
||||
chris,-1
|
||||
|
Can't render this file because it contains an unexpected character in line 6 and column 3.
|
@@ -0,0 +1,19 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
alice,10.1
|
||||
bob,29.2
|
||||
chris,-1.3
|
||||
|
Can't render this file because it contains an unexpected character in line 6 and column 3.
|
@@ -0,0 +1,19 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
alice,10
|
||||
bob,29
|
||||
chris,-1
|
||||
|
Can't render this file because it contains an unexpected character in line 6 and column 3.
|
@@ -0,0 +1,27 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require_relative "../version"
|
||||
|
||||
require "arrow"
|
||||
|
||||
require "pathname"
|
||||
require "tempfile"
|
||||
|
||||
require "test-unit"
|
||||
|
||||
require_relative "helper/fixture"
|
||||
@@ -0,0 +1,28 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Helper
|
||||
module Fixture
|
||||
def fixture_dir
|
||||
Pathname.new(__dir__).join("..", "fixture").expand_path
|
||||
end
|
||||
|
||||
def fixture_path(*components)
|
||||
fixture_dir.join(*components)
|
||||
end
|
||||
end
|
||||
end
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env ruby
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
ENV["TZ"] = "Asia/Tokyo"
|
||||
|
||||
$VERBOSE = true
|
||||
|
||||
require "pathname"
|
||||
|
||||
base_dir = Pathname.new(__dir__).parent.expand_path
|
||||
|
||||
lib_dir = base_dir + "lib"
|
||||
test_dir = base_dir + "test"
|
||||
|
||||
$LOAD_PATH.unshift(lib_dir.to_s)
|
||||
|
||||
require_relative "helper"
|
||||
|
||||
exit(Test::Unit::AutoRunner.run(true, test_dir.to_s))
|
||||
@@ -0,0 +1,112 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class ArrayBuilderTest < Test::Unit::TestCase
|
||||
sub_test_case(".build") do
|
||||
def assert_build(builder_class, raw_array)
|
||||
array = builder_class.build(raw_array)
|
||||
assert_equal(raw_array, array.to_a)
|
||||
end
|
||||
|
||||
sub_test_case("generic builder") do
|
||||
test("strings") do
|
||||
assert_build(Arrow::ArrayBuilder,
|
||||
["Hello", nil, "World"])
|
||||
end
|
||||
|
||||
test("boolean") do
|
||||
assert_build(Arrow::ArrayBuilder,
|
||||
[true, nil, false])
|
||||
end
|
||||
|
||||
test("positive integers") do
|
||||
assert_build(Arrow::ArrayBuilder,
|
||||
[1, nil, 2, nil, 3])
|
||||
end
|
||||
|
||||
test("negative integers") do
|
||||
assert_build(Arrow::ArrayBuilder,
|
||||
[nil, -1, nil, -2, nil, -3])
|
||||
end
|
||||
|
||||
test("times") do
|
||||
assert_build(Arrow::ArrayBuilder,
|
||||
[Time.at(0), Time.at(1), Time.at(2)])
|
||||
end
|
||||
|
||||
test("dates") do
|
||||
assert_build(Arrow::ArrayBuilder,
|
||||
[Date.new(2018, 1, 4), Date.new(2018, 1, 5)])
|
||||
end
|
||||
|
||||
test("datetimes") do
|
||||
assert_build(Arrow::ArrayBuilder,
|
||||
[
|
||||
DateTime.new(2018, 1, 4, 23, 18, 23),
|
||||
DateTime.new(2018, 1, 5, 0, 23, 21),
|
||||
])
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("specific builder") do
|
||||
test("empty") do
|
||||
assert_build(Arrow::Int32ArrayBuilder,
|
||||
[])
|
||||
end
|
||||
|
||||
test("values") do
|
||||
assert_build(Arrow::Int32ArrayBuilder,
|
||||
[1, -2])
|
||||
end
|
||||
|
||||
test("values, nils") do
|
||||
assert_build(Arrow::Int32ArrayBuilder,
|
||||
[1, -2, nil, nil])
|
||||
end
|
||||
|
||||
test("values, nils, values") do
|
||||
assert_build(Arrow::Int32ArrayBuilder,
|
||||
[1, -2, nil, nil, 3, -4])
|
||||
end
|
||||
|
||||
test("values, nils, values, nils") do
|
||||
assert_build(Arrow::Int32ArrayBuilder,
|
||||
[1, -2, nil, nil, 3, -4, nil, nil])
|
||||
end
|
||||
|
||||
test("nils") do
|
||||
assert_build(Arrow::Int32ArrayBuilder,
|
||||
[nil, nil])
|
||||
end
|
||||
|
||||
test("nils, values") do
|
||||
assert_build(Arrow::Int32ArrayBuilder,
|
||||
[nil, nil, 3, -4])
|
||||
end
|
||||
|
||||
test("nils, values, nil") do
|
||||
assert_build(Arrow::Int32ArrayBuilder,
|
||||
[nil, nil, 3, -4, nil, nil])
|
||||
end
|
||||
|
||||
test("nils, values, nil, values") do
|
||||
assert_build(Arrow::Int32ArrayBuilder,
|
||||
[nil, nil, 3, -4, nil, nil, 5, -6])
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,38 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class ArrayTest < Test::Unit::TestCase
|
||||
sub_test_case(".new") do
|
||||
test("Boolean") do
|
||||
array = Arrow::BooleanArray.new([true, false, true])
|
||||
assert_equal([true, false, true],
|
||||
array.to_a)
|
||||
end
|
||||
end
|
||||
|
||||
test("#each") do
|
||||
array = Arrow::BooleanArray.new([true, false, nil, true])
|
||||
assert_equal([true, false, nil, true],
|
||||
array.to_a)
|
||||
end
|
||||
|
||||
test("#[]") do
|
||||
array = Arrow::BooleanArray.new([true, false, nil, true])
|
||||
assert_equal([true, false, nil, true],
|
||||
[array[0], array[1], array[2], array[3]])
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,65 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class ChunkedArrayTest < Test::Unit::TestCase
|
||||
test("#each") do
|
||||
arrays = [
|
||||
Arrow::BooleanArray.new([true, false]),
|
||||
Arrow::BooleanArray.new([nil, true]),
|
||||
]
|
||||
chunked_array = Arrow::ChunkedArray.new(arrays)
|
||||
assert_equal([true, false, nil, true],
|
||||
chunked_array.to_a)
|
||||
end
|
||||
|
||||
sub_test_case("#pack") do
|
||||
test("basic array") do
|
||||
arrays = [
|
||||
Arrow::BooleanArray.new([true, false]),
|
||||
Arrow::BooleanArray.new([nil, true]),
|
||||
]
|
||||
chunked_array = Arrow::ChunkedArray.new(arrays)
|
||||
packed_chunked_array = chunked_array.pack
|
||||
assert_equal([
|
||||
Arrow::BooleanArray,
|
||||
[true, false, nil, true],
|
||||
],
|
||||
[
|
||||
packed_chunked_array.class,
|
||||
packed_chunked_array.to_a,
|
||||
])
|
||||
end
|
||||
|
||||
test("TimestampArray") do
|
||||
type = Arrow::TimestampDataType.new(:nano)
|
||||
arrays = [
|
||||
Arrow::TimestampArrayBuilder.new(type).build([Time.at(0)]),
|
||||
Arrow::TimestampArrayBuilder.new(type).build([Time.at(1)]),
|
||||
]
|
||||
chunked_array = Arrow::ChunkedArray.new(arrays)
|
||||
packed_chunked_array = chunked_array.pack
|
||||
assert_equal([
|
||||
Arrow::TimestampArray,
|
||||
[Time.at(0), Time.at(1)],
|
||||
],
|
||||
[
|
||||
packed_chunked_array.class,
|
||||
packed_chunked_array.to_a,
|
||||
])
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,43 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class ColumnTest < Test::Unit::TestCase
|
||||
test("#each") do
|
||||
arrays = [
|
||||
Arrow::BooleanArray.new([true, false]),
|
||||
Arrow::BooleanArray.new([nil, true]),
|
||||
]
|
||||
chunked_array = Arrow::ChunkedArray.new(arrays)
|
||||
column = Arrow::Column.new(Arrow::Field.new("visible", :boolean),
|
||||
chunked_array)
|
||||
assert_equal([true, false, nil, true],
|
||||
column.to_a)
|
||||
end
|
||||
|
||||
test("#pack") do
|
||||
arrays = [
|
||||
Arrow::BooleanArray.new([true, false]),
|
||||
Arrow::BooleanArray.new([nil, true]),
|
||||
]
|
||||
chunked_array = Arrow::ChunkedArray.new(arrays)
|
||||
column = Arrow::Column.new(Arrow::Field.new("visible", :boolean),
|
||||
chunked_array)
|
||||
packed_column = column.pack
|
||||
assert_equal([1, [true, false, nil, true]],
|
||||
[packed_column.data.n_chunks, packed_column.to_a])
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,118 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class CSVLoaderTest < Test::Unit::TestCase
|
||||
include Helper::Fixture
|
||||
|
||||
def load_csv(input)
|
||||
Arrow::CSVLoader.load(input, skip_lines: /^#/)
|
||||
end
|
||||
|
||||
sub_test_case(".load") do
|
||||
test("String: data: with header") do
|
||||
data = fixture_path("with-header-float.csv").read
|
||||
assert_equal(<<-TABLE, load_csv(data).to_s)
|
||||
name score
|
||||
0 alice 10.100000
|
||||
1 bob 29.200000
|
||||
2 chris -1.300000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("String: data: without header") do
|
||||
data = fixture_path("without-header-float.csv").read
|
||||
assert_equal(<<-TABLE, load_csv(data).to_s)
|
||||
0 1
|
||||
0 alice 10.100000
|
||||
1 bob 29.200000
|
||||
2 chris -1.300000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("String: path: with header") do
|
||||
path = fixture_path("with-header-float.csv").to_s
|
||||
assert_equal(<<-TABLE, load_csv(path).to_s)
|
||||
name score
|
||||
0 alice 10.100000
|
||||
1 bob 29.200000
|
||||
2 chris -1.300000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("String: path: without header") do
|
||||
path = fixture_path("without-header-float.csv").to_s
|
||||
assert_equal(<<-TABLE, load_csv(path).to_s)
|
||||
0 1
|
||||
0 alice 10.100000
|
||||
1 bob 29.200000
|
||||
2 chris -1.300000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Pathname: with header") do
|
||||
path = fixture_path("with-header-float.csv")
|
||||
assert_equal(<<-TABLE, load_csv(path).to_s)
|
||||
name score
|
||||
0 alice 10.100000
|
||||
1 bob 29.200000
|
||||
2 chris -1.300000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Pathname: without header") do
|
||||
path = fixture_path("without-header-float.csv")
|
||||
assert_equal(<<-TABLE, load_csv(path).to_s)
|
||||
0 1
|
||||
0 alice 10.100000
|
||||
1 bob 29.200000
|
||||
2 chris -1.300000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("null: with double quote") do
|
||||
path = fixture_path("null-with-double-quote.csv").to_s
|
||||
assert_equal(<<-TABLE, load_csv(path).to_s)
|
||||
name score
|
||||
0 alice 10
|
||||
1 bob
|
||||
2 chris -1
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("null: without double quote") do
|
||||
path = fixture_path("null-without-double-quote.csv").to_s
|
||||
assert_equal(<<-TABLE, load_csv(path).to_s)
|
||||
name score
|
||||
0 alice 10
|
||||
1 bob
|
||||
2 chris -1
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("number: float, integer") do
|
||||
path = fixture_path("float-integer.csv").to_s
|
||||
assert_equal([2.9, 10, -1.1],
|
||||
load_csv(path)[:score].to_a)
|
||||
end
|
||||
|
||||
test("number: integer, float") do
|
||||
path = fixture_path("integer-float.csv").to_s
|
||||
assert_equal([10.0, 2.9, -1.1],
|
||||
load_csv(path)[:score].to_a)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,34 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class CSVReaderTest < Test::Unit::TestCase
|
||||
include Helper::Fixture
|
||||
|
||||
test("#read") do
|
||||
CSV.open(fixture_path("with-header.csv").to_s,
|
||||
headers: true,
|
||||
skip_lines: /^#/) do |csv|
|
||||
reader = Arrow::CSVReader.new(csv)
|
||||
assert_equal(<<-TABLE, reader.read.to_s)
|
||||
name score
|
||||
0 alice 10
|
||||
1 bob 29
|
||||
2 chris -1
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,24 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class Date32ArrayTest < Test::Unit::TestCase
|
||||
test("#[]") do
|
||||
n_days_since_epoch = 17406 # 2017-08-28
|
||||
array = Arrow::Date32Array.new([n_days_since_epoch])
|
||||
assert_equal(Date.new(2017, 8, 28), array[0])
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,25 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class Date64ArrayTest < Test::Unit::TestCase
|
||||
test("#[]") do
|
||||
n_msecs_since_epoch = 1503878400000 # 2017-08-28T00:00:00Z
|
||||
array = Arrow::Date64Array.new([n_msecs_since_epoch])
|
||||
assert_equal(DateTime.new(2017, 8, 28, 0, 0, 0),
|
||||
array[0])
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,156 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class GroupTest < Test::Unit::TestCase
|
||||
include Helper::Fixture
|
||||
|
||||
def setup
|
||||
raw_table = {
|
||||
:group_key1 => Arrow::UInt8Array.new([1, 1, 2, 3, 3, 3]),
|
||||
:group_key2 => Arrow::UInt8Array.new([1, 1, 1, 1, 2, 2]),
|
||||
:int => Arrow::Int32Array.new([-1, -2, nil, -4, -5, -6]),
|
||||
:uint => Arrow::UInt32Array.new([1, nil, 3, 4, 5, 6]),
|
||||
:float => Arrow::FloatArray.new([nil, 2.2, 3.3, 4.4, 5.5, 6.6]),
|
||||
:string => Arrow::StringArray.new(["a", "b", "c", nil, "e", "f"]),
|
||||
}
|
||||
@table = Arrow::Table.new(raw_table)
|
||||
end
|
||||
|
||||
sub_test_case("key") do
|
||||
test("Time") do
|
||||
time_values = [
|
||||
Time.parse("2018-01-29"),
|
||||
Time.parse("2018-01-30"),
|
||||
]
|
||||
raw_table = {
|
||||
:time => Arrow::ArrayBuilder.build(time_values),
|
||||
:int => Arrow::Int32Array.new([-1, -2]),
|
||||
}
|
||||
table = Arrow::Table.new(raw_table)
|
||||
assert_equal(<<-TABLE, table.group(:time).count.to_s)
|
||||
time int
|
||||
0 2018-01-29T00:00:00+09:00 1
|
||||
1 2018-01-30T00:00:00+09:00 1
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("#count") do
|
||||
test("single") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1).count.to_s)
|
||||
group_key1 group_key2 int uint float string
|
||||
0 1 2 2 1 1 2
|
||||
1 2 1 0 1 1 1
|
||||
2 3 3 3 3 3 2
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("multiple") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).count.to_s)
|
||||
group_key1 group_key2 int uint float string
|
||||
0 1 1 2 1 1 2
|
||||
1 2 1 0 1 1 1
|
||||
2 3 1 1 1 1 0
|
||||
3 3 2 2 2 2 2
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("#sum") do
|
||||
test("single") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1).sum.to_s)
|
||||
group_key1 group_key2 int uint float
|
||||
0 1 2 -3 1 2.200000
|
||||
1 2 1 0 3 3.300000
|
||||
2 3 5 -15 15 16.500000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("multiple") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).sum.to_s)
|
||||
group_key1 group_key2 int uint float
|
||||
0 1 1 -3 1 2.200000
|
||||
1 2 1 0 3 3.300000
|
||||
2 3 1 -4 4 4.400000
|
||||
3 3 2 -11 11 12.100000
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("#average") do
|
||||
test("single") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1).average.to_s)
|
||||
group_key1 group_key2 int uint float
|
||||
0 1 1.000000 -1.500000 1.000000 2.200000
|
||||
1 2 1.000000 0.000000 3.000000 3.300000
|
||||
2 3 1.666667 -5.000000 5.000000 5.500000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("multiple") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).average.to_s)
|
||||
group_key1 group_key2 int uint float
|
||||
0 1 1 -1.500000 1.000000 2.200000
|
||||
1 2 1 0.000000 3.000000 3.300000
|
||||
2 3 1 -4.000000 4.000000 4.400000
|
||||
3 3 2 -5.500000 5.500000 6.050000
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("#min") do
|
||||
test("single") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1).min.to_s)
|
||||
group_key1 group_key2 int uint float
|
||||
0 1 1 -2 1 2.200000
|
||||
1 2 1 3 3.300000
|
||||
2 3 1 -6 4 4.400000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("multiple") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).min.to_s)
|
||||
group_key1 group_key2 int uint float
|
||||
0 1 1 -2 1 2.200000
|
||||
1 2 1 3 3.300000
|
||||
2 3 1 -4 4 4.400000
|
||||
3 3 2 -6 5 5.500000
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("#max") do
|
||||
test("single") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1).max.to_s)
|
||||
group_key1 group_key2 int uint float
|
||||
0 1 1 -1 1 2.200000
|
||||
1 2 1 3 3.300000
|
||||
2 3 2 -4 6 6.600000
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("multiple") do
|
||||
assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).max.to_s)
|
||||
group_key1 group_key2 int uint float
|
||||
0 1 1 -1 1 2.200000
|
||||
1 2 1 3 3.300000
|
||||
2 3 1 -4 4 4.400000
|
||||
3 3 2 -5 6 6.600000
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,115 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class RecordBatchFileReaderTest < Test::Unit::TestCase
|
||||
test("write/read") do
|
||||
fields = [
|
||||
Arrow::Field.new("uint8", :uint8),
|
||||
Arrow::Field.new("uint16", :uint16),
|
||||
Arrow::Field.new("uint32", :uint32),
|
||||
Arrow::Field.new("uint64", :uint64),
|
||||
Arrow::Field.new("int8", :int8),
|
||||
Arrow::Field.new("int16", :int16),
|
||||
Arrow::Field.new("int32", :int32),
|
||||
Arrow::Field.new("int64", :int64),
|
||||
Arrow::Field.new("float", :float),
|
||||
Arrow::Field.new("double", :double),
|
||||
]
|
||||
schema = Arrow::Schema.new(fields)
|
||||
|
||||
tempfile = Tempfile.new(["batch", ".arrow"])
|
||||
Arrow::FileOutputStream.open(tempfile.path, false) do |output|
|
||||
Arrow::RecordBatchFileWriter.open(output, schema) do |writer|
|
||||
uints = [1, 2, 4, 8]
|
||||
ints = [1, -2, 4, -8]
|
||||
floats = [1.1, -2.2, 4.4, -8.8]
|
||||
columns = [
|
||||
Arrow::UInt8Array.new(uints),
|
||||
Arrow::UInt16Array.new(uints),
|
||||
Arrow::UInt32Array.new(uints),
|
||||
Arrow::UInt64Array.new(uints),
|
||||
Arrow::Int8Array.new(ints),
|
||||
Arrow::Int16Array.new(ints),
|
||||
Arrow::Int32Array.new(ints),
|
||||
Arrow::Int64Array.new(ints),
|
||||
Arrow::FloatArray.new(floats),
|
||||
Arrow::DoubleArray.new(floats),
|
||||
]
|
||||
|
||||
record_batch = Arrow::RecordBatch.new(schema, 4, columns)
|
||||
writer.write_record_batch(record_batch)
|
||||
end
|
||||
end
|
||||
|
||||
Arrow::MemoryMappedInputStream.open(tempfile.path) do |input|
|
||||
reader = Arrow::RecordBatchFileReader.new(input)
|
||||
reader.each do |record_batch|
|
||||
assert_equal([
|
||||
{
|
||||
"uint8" => 1,
|
||||
"uint16" => 1,
|
||||
"uint32" => 1,
|
||||
"uint64" => 1,
|
||||
"int8" => 1,
|
||||
"int16" => 1,
|
||||
"int32" => 1,
|
||||
"int64" => 1,
|
||||
"float" => 1.100000023841858,
|
||||
"double" => 1.1,
|
||||
},
|
||||
{
|
||||
"uint8" => 2,
|
||||
"uint16" => 2,
|
||||
"uint32" => 2,
|
||||
"uint64" => 2,
|
||||
"int8" => -2,
|
||||
"int16" => -2,
|
||||
"int32" => -2,
|
||||
"int64" => -2,
|
||||
"float" => -2.200000047683716,
|
||||
"double" => -2.2,
|
||||
},
|
||||
{
|
||||
"uint8" => 4,
|
||||
"uint16" => 4,
|
||||
"uint32" => 4,
|
||||
"uint64" => 4,
|
||||
"int8" => 4,
|
||||
"int16" => 4,
|
||||
"int32" => 4,
|
||||
"int64" => 4,
|
||||
"float" => 4.400000095367432,
|
||||
"double" => 4.4,
|
||||
},
|
||||
{
|
||||
"uint8" => 8,
|
||||
"uint16" => 8,
|
||||
"uint32" => 8,
|
||||
"uint64" => 8,
|
||||
"int8" => -8,
|
||||
"int16" => -8,
|
||||
"int32" => -8,
|
||||
"int64" => -8,
|
||||
"float" => -8.800000190734863,
|
||||
"double" => -8.8,
|
||||
},
|
||||
],
|
||||
record_batch.collect(&:to_h))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,57 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class RecordBatchTest < Test::Unit::TestCase
|
||||
sub_test_case(".each") do
|
||||
setup do
|
||||
fields = [
|
||||
Arrow::Field.new("count", :uint32),
|
||||
]
|
||||
@schema = Arrow::Schema.new(fields)
|
||||
@counts = Arrow::UInt32Array.new([1, 2, 4, 8])
|
||||
@record_batch = Arrow::RecordBatch.new(@schema, @counts.length, [@counts])
|
||||
end
|
||||
|
||||
test("default") do
|
||||
records = []
|
||||
@record_batch.each do |record|
|
||||
records << [record, record.index]
|
||||
end
|
||||
assert_equal([
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
[2, 2],
|
||||
[3, 3],
|
||||
],
|
||||
records.collect {|record, i| [record.index, i]})
|
||||
end
|
||||
|
||||
test("reuse_record: true") do
|
||||
records = []
|
||||
@record_batch.each(reuse_record: true) do |record|
|
||||
records << [record, record.index]
|
||||
end
|
||||
assert_equal([
|
||||
[3, 0],
|
||||
[3, 1],
|
||||
[3, 2],
|
||||
[3, 3],
|
||||
],
|
||||
records.collect {|record, i| [record.index, i]})
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,33 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class RollingWindowTest < Test::Unit::TestCase
|
||||
include Helper::Fixture
|
||||
|
||||
def setup
|
||||
raw_table = {
|
||||
:number => Arrow::Int32Array.new([1, -2, nil, 4, 6, 3]),
|
||||
}
|
||||
@table = Arrow::Table.new(raw_table)
|
||||
end
|
||||
|
||||
test("#lag") do
|
||||
assert_equal(<<-ARRAY.chomp, @table.window.lag(:number).to_s)
|
||||
[null, -3, null, null, 2, -3]
|
||||
ARRAY
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,446 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class SlicerTest < Test::Unit::TestCase
|
||||
def setup
|
||||
@count_field = Arrow::Field.new("count", :uint32)
|
||||
@visible_field = Arrow::Field.new("visible", :boolean)
|
||||
schema = Arrow::Schema.new([@count_field, @visible_field])
|
||||
count_arrays = [
|
||||
Arrow::UInt32Array.new([0, 1, 2]),
|
||||
Arrow::UInt32Array.new([4, 8, 16]),
|
||||
Arrow::UInt32Array.new([32, 64, nil]),
|
||||
Arrow::UInt32Array.new([256]),
|
||||
]
|
||||
visible_arrays = [
|
||||
Arrow::BooleanArray.new([nil, true, false, nil]),
|
||||
Arrow::BooleanArray.new([true]),
|
||||
Arrow::BooleanArray.new([true, false]),
|
||||
Arrow::BooleanArray.new([nil]),
|
||||
Arrow::BooleanArray.new([nil]),
|
||||
Arrow::BooleanArray.new([true]),
|
||||
]
|
||||
@count_array = Arrow::ChunkedArray.new(count_arrays)
|
||||
@visible_array = Arrow::ChunkedArray.new(visible_arrays)
|
||||
@count_column = Arrow::Column.new(@count_field, @count_array)
|
||||
@visible_column = Arrow::Column.new(@visible_field, @visible_array)
|
||||
@table = Arrow::Table.new(schema, [@count_column, @visible_column])
|
||||
end
|
||||
|
||||
sub_test_case("column") do
|
||||
test("BooleanArray") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 8 true
|
||||
2 16 true
|
||||
3 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("not BooleanArray") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.count
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 2 false
|
||||
2 4
|
||||
3 8 true
|
||||
4 16 true
|
||||
5 32 false
|
||||
6 64
|
||||
7 256 true
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("!column") do
|
||||
test("BooleanArray") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!slicer.visible
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 2 false
|
||||
1 32 false
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("not BooleanArray") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!slicer.count
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
test("column.null?") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible.null?
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
1 4
|
||||
2 64
|
||||
3
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("column.valid?") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible.valid?
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 2 false
|
||||
2 8 true
|
||||
3 16 true
|
||||
4 32 false
|
||||
5 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
sub_test_case("column ==") do
|
||||
test("nil") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible == nil
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
1 4
|
||||
2 64
|
||||
3
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("value") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible == true
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 8 true
|
||||
2 16 true
|
||||
3 256 true
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("!(column ==)") do
|
||||
test("nil") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!(slicer.visible == nil)
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 2 false
|
||||
2 8 true
|
||||
3 16 true
|
||||
4 32 false
|
||||
5 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("value") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!(slicer.visible == true)
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 2 false
|
||||
1 32 false
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("column !=") do
|
||||
test("nil") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible != nil
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 2 false
|
||||
2 8 true
|
||||
3 16 true
|
||||
4 32 false
|
||||
5 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("value") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible != true
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 2 false
|
||||
1 32 false
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
test("column < value") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.count < 16
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
1 1 true
|
||||
2 2 false
|
||||
3 4
|
||||
4 8 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("!(column < value)") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!(slicer.count < 16)
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 16 true
|
||||
1 32 false
|
||||
2 64
|
||||
3 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("column <= value") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.count <= 16
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
1 1 true
|
||||
2 2 false
|
||||
3 4
|
||||
4 8 true
|
||||
5 16 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("!(column <= value)") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!(slicer.count <= 16)
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 32 false
|
||||
1 64
|
||||
2 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("column > value") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.count > 16
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 32 false
|
||||
1 64
|
||||
2 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("!(column > value)") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!(slicer.count > 16)
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
1 1 true
|
||||
2 2 false
|
||||
3 4
|
||||
4 8 true
|
||||
5 16 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("column >= value") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.count >= 16
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 16 true
|
||||
1 32 false
|
||||
2 64
|
||||
3 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("!(column >= value)") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!(slicer.count >= 16)
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
1 1 true
|
||||
2 2 false
|
||||
3 4
|
||||
4 8 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("column.in") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.count.in?([1, 4, 16, 64])
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 4
|
||||
2 16 true
|
||||
3 64
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("!column.in") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!slicer.count.in?([1, 4, 16, 64])
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
1 2 false
|
||||
2 8 true
|
||||
3 32 false
|
||||
4 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("condition & condition") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible & (slicer.count >= 16)
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 16 true
|
||||
1 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("condition | condition") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible | (slicer.count >= 16)
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 8 true
|
||||
2 16 true
|
||||
3 32 false
|
||||
4 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("condition ^ condition") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible ^ (slicer.count >= 16)
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 8 true
|
||||
2 32 false
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("select") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible.select do |value|
|
||||
value.nil? or value
|
||||
end
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
1 1 true
|
||||
2 4
|
||||
3 8 true
|
||||
4 16 true
|
||||
5 64
|
||||
6
|
||||
7 256 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("!select") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!slicer.visible.select do |value|
|
||||
value.nil? or value
|
||||
end
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 2 false
|
||||
1 32 false
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("reject") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
slicer.visible.reject do |value|
|
||||
value.nil? or value
|
||||
end
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 2 false
|
||||
1 32 false
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("!reject") do
|
||||
sliced_table = @table.slice do |slicer|
|
||||
!slicer.visible.reject do |value|
|
||||
value.nil? or value
|
||||
end
|
||||
end
|
||||
assert_equal(<<-TABLE, sliced_table.to_s)
|
||||
count visible
|
||||
0 0
|
||||
1 1 true
|
||||
2 4
|
||||
3 8 true
|
||||
4 16 true
|
||||
5 64
|
||||
6
|
||||
7 256 true
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,464 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class TableTest < Test::Unit::TestCase
|
||||
include Helper::Fixture
|
||||
|
||||
def setup
|
||||
@count_field = Arrow::Field.new("count", :uint8)
|
||||
@visible_field = Arrow::Field.new("visible", :boolean)
|
||||
schema = Arrow::Schema.new([@count_field, @visible_field])
|
||||
count_arrays = [
|
||||
Arrow::UInt8Array.new([1, 2]),
|
||||
Arrow::UInt8Array.new([4, 8, 16]),
|
||||
Arrow::UInt8Array.new([32, 64]),
|
||||
Arrow::UInt8Array.new([128]),
|
||||
]
|
||||
visible_arrays = [
|
||||
Arrow::BooleanArray.new([true, false, nil]),
|
||||
Arrow::BooleanArray.new([true]),
|
||||
Arrow::BooleanArray.new([true, false]),
|
||||
Arrow::BooleanArray.new([nil]),
|
||||
Arrow::BooleanArray.new([nil]),
|
||||
]
|
||||
@count_array = Arrow::ChunkedArray.new(count_arrays)
|
||||
@visible_array = Arrow::ChunkedArray.new(visible_arrays)
|
||||
@count_column = Arrow::Column.new(@count_field, @count_array)
|
||||
@visible_column = Arrow::Column.new(@visible_field, @visible_array)
|
||||
@table = Arrow::Table.new(schema, [@count_column, @visible_column])
|
||||
end
|
||||
|
||||
test("#columns") do
|
||||
assert_equal(["count", "visible"],
|
||||
@table.columns.collect(&:name))
|
||||
end
|
||||
|
||||
sub_test_case("#slice") do
|
||||
test("Arrow::BooleanArray") do
|
||||
target_rows_raw = [nil, true, true, false, true, false, true, true]
|
||||
target_rows = Arrow::BooleanArray.new(target_rows_raw)
|
||||
assert_equal(<<-TABLE, @table.slice(target_rows).to_s)
|
||||
count visible
|
||||
0 2 false
|
||||
1 4
|
||||
2 16 true
|
||||
3 64
|
||||
4 128
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Array: boolean") do
|
||||
target_rows_raw = [nil, true, true, false, true, false, true, true]
|
||||
assert_equal(<<-TABLE, @table.slice(target_rows_raw).to_s)
|
||||
count visible
|
||||
0 2 false
|
||||
1 4
|
||||
2 16 true
|
||||
3 64
|
||||
4 128
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Integer: positive") do
|
||||
assert_equal(<<-TABLE, @table.slice(2).to_s)
|
||||
count visible
|
||||
0 4
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Integer: negative") do
|
||||
assert_equal(<<-TABLE, @table.slice(-1).to_s)
|
||||
count visible
|
||||
0 128
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Range: positive: include end") do
|
||||
assert_equal(<<-TABLE, @table.slice(2..4).to_s)
|
||||
count visible
|
||||
0 4
|
||||
1 8 true
|
||||
2 16 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Range: positive: exclude end") do
|
||||
assert_equal(<<-TABLE, @table.slice(2...4).to_s)
|
||||
count visible
|
||||
0 4
|
||||
1 8 true
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Range: negative: include end") do
|
||||
assert_equal(<<-TABLE, @table.slice(-4..-2).to_s)
|
||||
count visible
|
||||
0 16 true
|
||||
1 32 false
|
||||
2 64
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Range: negative: exclude end") do
|
||||
assert_equal(<<-TABLE, @table.slice(-4...-2).to_s)
|
||||
count visible
|
||||
0 16 true
|
||||
1 32 false
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("[from, to]: positive") do
|
||||
assert_equal(<<-TABLE, @table.slice(0, 2).to_s)
|
||||
count visible
|
||||
0 1 true
|
||||
1 2 false
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("[from, to]: negative") do
|
||||
assert_equal(<<-TABLE, @table.slice(-4, 2).to_s)
|
||||
count visible
|
||||
0 16 true
|
||||
1 32 false
|
||||
TABLE
|
||||
end
|
||||
|
||||
sub_test_case("wrong argument") do
|
||||
test("no arguments") do
|
||||
message = "wrong number of arguments (given 0, expected 1..2)"
|
||||
assert_raise(ArgumentError.new(message)) do
|
||||
@table.slice
|
||||
end
|
||||
end
|
||||
|
||||
test("too many arguments: with block") do
|
||||
message = "wrong number of arguments (given 3, expected 1..2)"
|
||||
assert_raise(ArgumentError.new(message)) do
|
||||
@table.slice(1, 2, 3)
|
||||
end
|
||||
end
|
||||
|
||||
test("too many arguments: without block") do
|
||||
message = "wrong number of arguments (given 3, expected 0..2)"
|
||||
assert_raise(ArgumentError.new(message)) do
|
||||
@table.slice(1, 2, 3) {}
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("#[]") do
|
||||
test("[String]") do
|
||||
assert_equal(@count_column, @table["count"])
|
||||
end
|
||||
|
||||
test("[Symbol]") do
|
||||
assert_equal(@visible_column, @table[:visible])
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("#merge") do
|
||||
sub_test_case("Hash") do
|
||||
test("add") do
|
||||
name_array = Arrow::StringArray.new(["a", "b", "c", "d", "e", "f", "g", "h"])
|
||||
assert_equal(<<-TABLE, @table.merge(:name => name_array).to_s)
|
||||
count visible name
|
||||
0 1 true a
|
||||
1 2 false b
|
||||
2 4 c
|
||||
3 8 true d
|
||||
4 16 true e
|
||||
5 32 false f
|
||||
6 64 g
|
||||
7 128 h
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("remove") do
|
||||
assert_equal(<<-TABLE, @table.merge(:visible => nil).to_s)
|
||||
count
|
||||
0 1
|
||||
1 2
|
||||
2 4
|
||||
3 8
|
||||
4 16
|
||||
5 32
|
||||
6 64
|
||||
7 128
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("replace") do
|
||||
visible_array = Arrow::Int32Array.new([1] * @visible_array.length)
|
||||
assert_equal(<<-TABLE, @table.merge(:visible => visible_array).to_s)
|
||||
count visible
|
||||
0 1 1
|
||||
1 2 1
|
||||
2 4 1
|
||||
3 8 1
|
||||
4 16 1
|
||||
5 32 1
|
||||
6 64 1
|
||||
7 128 1
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("Arrow::Table") do
|
||||
test("add") do
|
||||
name_array = Arrow::StringArray.new(["a", "b", "c", "d", "e", "f", "g", "h"])
|
||||
table = Arrow::Table.new("name" => name_array)
|
||||
assert_equal(<<-TABLE, @table.merge(table).to_s)
|
||||
count visible name
|
||||
0 1 true a
|
||||
1 2 false b
|
||||
2 4 c
|
||||
3 8 true d
|
||||
4 16 true e
|
||||
5 32 false f
|
||||
6 64 g
|
||||
7 128 h
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("replace") do
|
||||
visible_array = Arrow::Int32Array.new([1] * @visible_array.length)
|
||||
table = Arrow::Table.new("visible" => visible_array)
|
||||
assert_equal(<<-TABLE, @table.merge(table).to_s)
|
||||
count visible
|
||||
0 1 1
|
||||
1 2 1
|
||||
2 4 1
|
||||
3 8 1
|
||||
4 16 1
|
||||
5 32 1
|
||||
6 64 1
|
||||
7 128 1
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
test("column name getter") do
|
||||
assert_equal(@visible_column, @table.visible)
|
||||
end
|
||||
|
||||
sub_test_case("#remove_column") do
|
||||
test("String") do
|
||||
assert_equal(<<-TABLE, @table.remove_column("visible").to_s)
|
||||
count
|
||||
0 1
|
||||
1 2
|
||||
2 4
|
||||
3 8
|
||||
4 16
|
||||
5 32
|
||||
6 64
|
||||
7 128
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("Symbol") do
|
||||
assert_equal(<<-TABLE, @table.remove_column(:visible).to_s)
|
||||
count
|
||||
0 1
|
||||
1 2
|
||||
2 4
|
||||
3 8
|
||||
4 16
|
||||
5 32
|
||||
6 64
|
||||
7 128
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("unknown column name") do
|
||||
assert_raise(KeyError) do
|
||||
@table.remove_column(:nonexistent)
|
||||
end
|
||||
end
|
||||
|
||||
test("Integer") do
|
||||
assert_equal(<<-TABLE, @table.remove_column(1).to_s)
|
||||
count
|
||||
0 1
|
||||
1 2
|
||||
2 4
|
||||
3 8
|
||||
4 16
|
||||
5 32
|
||||
6 64
|
||||
7 128
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("negative integer") do
|
||||
assert_equal(<<-TABLE, @table.remove_column(-1).to_s)
|
||||
count
|
||||
0 1
|
||||
1 2
|
||||
2 4
|
||||
3 8
|
||||
4 16
|
||||
5 32
|
||||
6 64
|
||||
7 128
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("too small index") do
|
||||
assert_raise(IndexError) do
|
||||
@table.remove_column(-3)
|
||||
end
|
||||
end
|
||||
|
||||
test("too large index") do
|
||||
assert_raise(IndexError) do
|
||||
@table.remove_column(2)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("#select_columns") do
|
||||
def setup
|
||||
raw_table = {
|
||||
:a => Arrow::UInt8Array.new([1]),
|
||||
:b => Arrow::UInt8Array.new([1]),
|
||||
:c => Arrow::UInt8Array.new([1]),
|
||||
:d => Arrow::UInt8Array.new([1]),
|
||||
:e => Arrow::UInt8Array.new([1]),
|
||||
}
|
||||
@table = Arrow::Table.new(raw_table)
|
||||
end
|
||||
|
||||
test("names") do
|
||||
assert_equal(<<-TABLE, @table.select_columns(:c, :a).to_s)
|
||||
c a
|
||||
0 1 1
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("range") do
|
||||
assert_equal(<<-TABLE, @table.select_columns(2...4).to_s)
|
||||
c d
|
||||
0 1 1
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("indexes") do
|
||||
assert_equal(<<-TABLE, @table.select_columns(0, -1, 2).to_s)
|
||||
a e c
|
||||
0 1 1 1
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("mixed") do
|
||||
assert_equal(<<-TABLE, @table.select_columns(:a, -1, 2..3).to_s)
|
||||
a e c d
|
||||
0 1 1 1 1
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("block") do
|
||||
selected_table = @table.select_columns.with_index do |column, i|
|
||||
column.name == "a" or i.odd?
|
||||
end
|
||||
assert_equal(<<-TABLE, selected_table.to_s)
|
||||
a b d
|
||||
0 1 1 1
|
||||
TABLE
|
||||
end
|
||||
|
||||
test("names, indexes and block") do
|
||||
selected_table = @table.select_columns(:a, -1) do |column|
|
||||
column.name == "a"
|
||||
end
|
||||
assert_equal(<<-TABLE, selected_table.to_s)
|
||||
a
|
||||
0 1
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
|
||||
sub_test_case("#save and .load") do
|
||||
sub_test_case(":format") do
|
||||
test("default") do
|
||||
file = Tempfile.new(["red-arrow", ".arrow"])
|
||||
@table.save(file.path)
|
||||
assert_equal(@table, Arrow::Table.load(file.path))
|
||||
end
|
||||
|
||||
test(":batch") do
|
||||
file = Tempfile.new(["red-arrow", ".arrow"])
|
||||
@table.save(file.path, :format => :batch)
|
||||
assert_equal(@table, Arrow::Table.load(file.path, :format => :batch))
|
||||
end
|
||||
|
||||
test(":stream") do
|
||||
file = Tempfile.new(["red-arrow", ".arrow"])
|
||||
@table.save(file.path, :format => :stream)
|
||||
assert_equal(@table, Arrow::Table.load(file.path, :format => :stream))
|
||||
end
|
||||
|
||||
test(":csv") do
|
||||
file = Tempfile.new(["red-arrow", ".csv"])
|
||||
@table.save(file.path, :format => :csv)
|
||||
assert_equal(@table, Arrow::Table.load(file.path, :format => :csv))
|
||||
end
|
||||
|
||||
sub_test_case("load: auto detect") do
|
||||
test(":batch") do
|
||||
file = Tempfile.new(["red-arrow", ".arrow"])
|
||||
@table.save(file.path, :format => :batch)
|
||||
assert_equal(@table, Arrow::Table.load(file.path))
|
||||
end
|
||||
|
||||
test(":stream") do
|
||||
file = Tempfile.new(["red-arrow", ".arrow"])
|
||||
@table.save(file.path, :format => :stream)
|
||||
assert_equal(@table, Arrow::Table.load(file.path))
|
||||
end
|
||||
|
||||
test(":csv") do
|
||||
path = fixture_path("with-header.csv")
|
||||
assert_equal(<<-TABLE, Arrow::Table.load(path, skip_lines: /^#/).to_s)
|
||||
name score
|
||||
0 alice 10
|
||||
1 bob 29
|
||||
2 chris -1
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
test("#pack") do
|
||||
packed_table = @table.pack
|
||||
column_n_chunks = packed_table.columns.collect {|c| c.data.n_chunks}
|
||||
assert_equal([[1, 1], <<-TABLE], [column_n_chunks, packed_table.to_s])
|
||||
count visible
|
||||
0 1 true
|
||||
1 2 false
|
||||
2 4
|
||||
3 8 true
|
||||
4 16 true
|
||||
5 32 false
|
||||
6 64
|
||||
7 128
|
||||
TABLE
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,26 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class TimestampArrayTest < Test::Unit::TestCase
|
||||
test("#[]") do
|
||||
sec = 1513267750
|
||||
usec = 914509
|
||||
array = Arrow::TimestampArray.new(:micro, [sec * (10 ** 6) + usec])
|
||||
time = Time.at(sec, usec)
|
||||
assert_equal(time, array[0])
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,71 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
require "pathname"
|
||||
|
||||
version_rb_path = Pathname.new(__FILE__)
|
||||
base_dir = version_rb_path.dirname
|
||||
pom_xml_path = base_dir.join("..", "..", "java", "pom.xml")
|
||||
lib_version_rb_path = base_dir.join("lib", "arrow", "version.rb")
|
||||
|
||||
need_update = false
|
||||
if not lib_version_rb_path.exist?
|
||||
need_update = true
|
||||
elsif version_rb_path.mtime > lib_version_rb_path.mtime
|
||||
need_update = true
|
||||
elsif pom_xml_path.exist? and pom_xml_path.mtime > lib_version_rb_path.mtime
|
||||
need_update = true
|
||||
end
|
||||
|
||||
if need_update
|
||||
version = pom_xml_path.read.scan(/^ <version>(.+?)<\/version>/)[0][0]
|
||||
major, minor, micro, tag = version.split(/[.-]/)
|
||||
lib_version_rb_path.open("w") do |lib_version_rb|
|
||||
lib_version_rb.puts(<<-RUBY)
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
module Arrow
|
||||
module Version
|
||||
MAJOR = #{major}
|
||||
MINOR = #{minor}
|
||||
MICRO = #{micro}
|
||||
TAG = #{tag ? tag.dump : nil}
|
||||
STRING = #{version.dump}
|
||||
end
|
||||
|
||||
VERSION = Version::STRING
|
||||
end
|
||||
RUBY
|
||||
end
|
||||
end
|
||||
|
||||
require_relative "lib/arrow/version"
|
||||
Reference in new issue
Block a user