mirror of
https://github.com/apache/arrow.git
synced 2026-10-29 09:05:47 +08:00
initial work on JSON reader python wrapper
This commit is contained in:
1 parent
184b8deb65
commit
4edb201f62
2 files changed
+110
No files matched your search
@@ -0,0 +1,82 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# cython: profile=False
|
||||
# distutils: language = c++
|
||||
# cython: embedsignature = True
|
||||
# cython: language_level = 3
|
||||
|
||||
from pyarrow.includes.common cimport *
|
||||
from pyarrow.includes.libarrow cimport *
|
||||
from pyarrow.lib cimport (check_status)
|
||||
|
||||
|
||||
cdef class ReadOptions:
|
||||
|
||||
cdef:
|
||||
CJSONReadOptions options
|
||||
|
||||
# Avoid mistakingly creating attributes
|
||||
__slots__ = ()
|
||||
|
||||
def __init__(self, use_threads=None, block_size=None):
|
||||
self.options = CJSONReadOptions.Defaults()
|
||||
if use_threads is not None:
|
||||
self.use_threads = use_threads
|
||||
if block_size is not None:
|
||||
self.block_size = block_size
|
||||
|
||||
@property
|
||||
def use_threads(self):
|
||||
"""
|
||||
Whether to use multiple threads to accelerate reading.
|
||||
"""
|
||||
return self.options.use_threads
|
||||
|
||||
@use_threads.setter
|
||||
def use_threads(self, value):
|
||||
self.options.use_threads = value
|
||||
|
||||
@property
|
||||
def block_size(self):
|
||||
"""
|
||||
How much bytes to process at a time from the input stream.
|
||||
This will determine multi-threading granularity as well as
|
||||
the size of individual chunks in the Table.
|
||||
"""
|
||||
return self.options.block_size
|
||||
|
||||
@block_size.setter
|
||||
def block_size(self, value):
|
||||
self.options.block_size = value
|
||||
|
||||
|
||||
cdef _get_read_options(ReadOptions read_options, CCSVReadOptions* out):
|
||||
if read_options is None:
|
||||
out[0] = CCSVReadOptions.Defaults()
|
||||
else:
|
||||
out[0] = read_options.options
|
||||
|
||||
|
||||
def parse_json(input_file, read_options=None, parse_options=None,
|
||||
convert_options=None, MemoryPool memory_pool=None):
|
||||
cdef:
|
||||
CCSVReadOptions c_read_options
|
||||
|
||||
_get_read_options(read_options, &c_read_options)
|
||||
|
||||
check_status(ParseOne())
|
||||
@@ -1045,6 +1045,34 @@ cdef extern from "arrow/csv/api.h" namespace "arrow::csv" nogil:
|
||||
CStatus Read(shared_ptr[CTable]* out)
|
||||
|
||||
|
||||
cdef extern from "arrow/json/options.h" nogil:
|
||||
|
||||
cdef cppclass CJSONReadOptions" arrow::json::ReadOptions":
|
||||
c_bool use_threads
|
||||
int32_t block_size
|
||||
|
||||
@staticmethod
|
||||
CJSONReadOptions Defaults()
|
||||
|
||||
cdef cppclass CJSONParseOptions" arrow::json::ParseOptions":
|
||||
shared_ptr[CSchema] explicit_schema
|
||||
c_bool newlines_in_values
|
||||
|
||||
@staticmethod
|
||||
CJSONParseOptions Defaults()
|
||||
|
||||
|
||||
cdef extern from "arrow/json/reader.h" namespace "arrow::json" nogil:
|
||||
|
||||
cdef cppclass CJSONReader" arrow::json::TableReader":
|
||||
@staticmethod
|
||||
CStatus Make(CMemoryPool*, shared_ptr[InputStream],
|
||||
CJSONReadOptions, CJSONParseOptions,
|
||||
shared_ptr[CJSONReader]* out)
|
||||
|
||||
CStatus Read(shared_ptr[CTable]* out)
|
||||
|
||||
|
||||
cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
|
||||
|
||||
cdef cppclass CFunctionContext" arrow::compute::FunctionContext":
|
||||
|
||||
Reference in new issue
Block a user