mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-29 09:05:47 +08:00
Code Activity

PartitionScheme [skip ci]

Browse Source
This commit is contained in:
Krisztián Szűcs committed 2019-12-12 12:26:41 +01:00
1 parent bd6e1d656d
commit d14cf502b6
3 files changed
+64 -86
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

python/pyarrow/_dataset.pyx
+50 -25
View File
@@ -54,38 +54,14 @@ cdef class FileScanOptions(ScanOptions):
pass
cdef class CsvScanOptions(FileScanOptions):
pass
cdef class FeatherScanOptions(FileScanOptions):
pass
cdef class ParquetScanOptions(FileScanOptions):
pass
cdef class JsonScanOptions(FileScanOptions):
pass
cdef class FileWriteOptions(WriteOptions):
pass
cdef class CsvWriteOptions(FileWriteOptions):
pass
cdef class FeatherWriterOptions(FileWriteOptions):
pass
cdef class JsonWriterOptions(FileWriteOptions):
pass
cdef class ParquetWriterOptions(FileWriteOptions):
pass
@@ -111,6 +87,55 @@ cdef class ParquetFileFormat(FileFormat):
pass
cdef class PartitionScheme:
cdef:
shared_ptr[CPartitionScheme] wrapped
CPartitionScheme* scheme
cdef init(self, const shared_ptr[CPartitionScheme]& sp):
self.wrapped = sp
self.scheme = sp.get()
@staticmethod
cdef wrap(shared_ptr[CPartitionScheme]& sp):
cdef PartitionScheme self
# TODO(kszucs): either choose type() or name() but consistently
typ = frombytes(sp.get().name())
if typ == 'schema_partition_scheme':
self = SchemaPartitionScheme.__new__(SchemaPartitionScheme)
else:
raise TypeError(typ)
self.init(sp)
return self
cdef inline shared_ptr[CPartitionScheme] unwrap(self):
return self.wrapped
cdef class SchemaPartitionScheme(PartitionScheme):
cdef:
CSchemaPartitionScheme* schema_scheme # hmmm...
def __init__(self, Schema schema):
cdef shared_ptr[CSchemaPartitionScheme] scheme
scheme = make_shared[CSchemaPartitionScheme](
pyarrow_unwrap_schema(schema)
)
self.init(<shared_ptr[CPartitionScheme]> scheme)
cdef init(self, const shared_ptr[CPartitionScheme]& sp):
PartitionScheme.init(self, sp)
self.schema_scheme = <CSchemaPartitionScheme*> sp.get()
@property
def schema(self):
return pyarrow_wrap_schema(self.schema_scheme.schema())
cdef class FileSystemDiscoveryOptions:
cdef:
@@ -369,7 +394,7 @@ cdef class DataSource:
elif typ == 'filesystem_data_source':
self = FileSystemDataSource.__new__(FileSystemDataSource)
else:
self = DataSource.__new__(DataSource)
raise TypeError(typ)
self.init(sp)
return self
python/pyarrow/includes/libarrow_dataset.pxd
+14 -59
View File
@@ -173,12 +173,12 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
cdef cppclass CFileBasedDataFragment \
"arrow::dataset::FileBasedDataFragment"(CDataFragment):
CFileBasedDataFragment(const CFileSource& source,
shared_ptr[CFileFormat] format,
CFileFormatPtr format,
CScanOptionsPtr scan_options)
CStatus Scan(CScanContextPtr scan_context,
shared_ptr[CScanTaskIterator]* out)
const CFileSource& source()
shared_ptr[CFileFormat] format()
CFileFormatPtr format()
CScanOptionsPtr scan_options()
cdef cppclass CFileSystemDataSource \
@@ -186,68 +186,12 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
@staticmethod
CStatus Make(CFileSystem* filesystem,
const CSelector& selector,
shared_ptr[CFileFormat] format,
CFileFormatPtr format,
CScanOptionsPtr scan_options,
shared_ptr[CFileSystemDataSource]* out)
c_string type()
shared_ptr[CDataFragmentIterator] GetFragments(CScanOptionsPtr options)
############################### File CSV ##################################
cdef cppclass CCsvScanOptions "arrow::dataset::CsvScanOptions"(
CFileScanOptions):
c_string file_type()
cdef cppclass CCsvWriteOptions "arrow::dataset::CsvWriteOptions"(
CFileWriteOptions):
c_string file_type()
cdef cppclass CCsvFileFormat" arrow::dataset::CsvFileFormat"(CFileFormat):
c_string name()
c_bool IsKnownExtension(const c_string& ext)
CStatus ScanFile(const CFileSource& source,
CScanOptionsPtr scan_options,
CScanContextPtr scan_context,
shared_ptr[CScanTaskIterator]* out)
############################### File JSON ################################
cdef cppclass CJsonScanOptions "arrow::dataset::JsonScanOptions"(
CFileScanOptions):
c_string file_type()
cdef cppclass CJsonWriteOptions "arrow::dataset::JsonWriteOptions"(
CFileWriteOptions):
c_string file_type()
cdef cppclass CJsonFileFormat "arrow::dataset::JsonFileFormat"(
CFileFormat):
c_string name()
c_bool IsKnownExtension(const c_string& ext)
CStatus ScanFile(const CFileSource& source,
shared_ptr[CScanOptions] scan_options,
shared_ptr[CScanContext] scan_context,
shared_ptr[CScanTaskIterator]* out)
############################### File Feather #############################
cdef cppclass CFeatherScanOptions "arrow::dataset::FeatherScanOptions"(
CFileScanOptions):
c_string file_type()
cdef cppclass CFeatherWriterOptions "arrow::dataset::FeatherWriterOptions"(
CFileWriteOptions):
c_string file_type()
cdef cppclass CFeatherFileFormat "arrow::dataset::FeatherFileFormat"(
CFileFormat):
c_string name()
c_bool IsKnownExtension(const c_string& ext)
CStatus ScanFile(const CFileSource& source,
shared_ptr[CScanOptions] scan_options,
shared_ptr[CScanContext] scan_context,
shared_ptr[CScanTaskIterator]* out)
############################### File Parquet #############################
cdef cppclass CParquetScanOptions "arrow::dataset::ParquetScanOptions"(
@@ -306,3 +250,14 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
CSelector,
CFileFormatPtr format,
CFileSystemDiscoveryOptions options)
############################### Partitioning ##############################
cdef cppclass CPartitionScheme "arrow::dataset::PartitionScheme":
c_string name() const
# CResult[CExpressionPtr] Parse(const c_string& path) const
cdef cppclass CSchemaPartitionScheme "arrow::dataset::SchemaPartitionScheme"(CPartitionScheme):
CSchemaPartitionScheme(shared_ptr[CSchema] schema)
c_string name() const
const shared_ptr[CSchema]& schema()
python/pyarrow/tests/test_dataset.py
-2
View File
@@ -213,5 +213,3 @@ def test_file_system_discovery(fs):
selector = FileSelector('/base', recursive=True)
fileformat = ParquetFileFormat()
discovery = FileSystemDataSourceDiscovery(fs, selector, fileformat)