mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-29 00:56:31 +08:00
Code Activity

Add flake8 file for Cython, fix Cython style errors

Browse Source 
Change-Id: Idb47b3f6e8cc9507574a68575f40527eb1a72cdd
This commit is contained in:
Wes McKinney committed 2017-08-01 15:43:21 -04:00
1 parent 7ffa6135cc
commit c0af17c370
12 files changed
+80 -82
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

python/.flake8.cython
+1 -1
View File
@@ -17,4 +17,4 @@
[flake8]
filename = *.pyx,*.px*
ignore = E211, E901,E225,E226,E227
ignore = E211,E901,E225,E226,E227
python/pyarrow/includes/libarrow.pxd
+34 -36
View File
@@ -178,7 +178,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
cdef cppclass CStringType" arrow::StringType"(CDataType):
pass
cdef cppclass CFixedSizeBinaryType" arrow::FixedSizeBinaryType"(CFixedWidthType):
cdef cppclass CFixedSizeBinaryType\
" arrow::FixedSizeBinaryType"(CFixedWidthType):
CFixedSizeBinaryType(int byte_width)
int byte_width()
int bit_width()
@@ -209,7 +210,6 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
const shared_ptr[CKeyValueMetadata]& metadata)
shared_ptr[CField] RemoveMetadata()
cdef cppclass CStructType" arrow::StructType"(CDataType):
CStructType(const vector[shared_ptr[CField]]& fields)
@@ -309,9 +309,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
cdef cppclass CStructArray" arrow::StructArray"(CArray):
CStructArray(shared_ptr[CDataType] type, int64_t length,
vector[shared_ptr[CArray]] children,
shared_ptr[CBuffer] null_bitmap = nullptr, int64_t null_count = 0,
int64_t offset = 0)
vector[shared_ptr[CArray]] children,
shared_ptr[CBuffer] null_bitmap=nullptr,
int64_t null_count=0,
int64_t offset=0)
shared_ptr[CArray] field(int pos)
const vector[shared_ptr[CArray]] fields()
@@ -462,7 +463,6 @@ cdef extern from "arrow/io/interfaces.h" namespace "arrow::io" nogil:
cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
cdef cppclass FileOutputStream(OutputStream):
@staticmethod
CStatus Open(const c_string& path, shared_ptr[FileOutputStream]* file)
@@ -479,12 +479,12 @@ cdef extern from "arrow/io/file.h" namespace "arrow::io" nogil:
int file_descriptor()
cdef cppclass CMemoryMappedFile" arrow::io::MemoryMappedFile"\
(ReadWriteFileInterface):
cdef cppclass CMemoryMappedFile\
" arrow::io::MemoryMappedFile"(ReadWriteFileInterface):
@staticmethod
CStatus Create(const c_string& path, int64_t size,
shared_ptr[CMemoryMappedFile]* file)
shared_ptr[CMemoryMappedFile]* file)
@staticmethod
CStatus Open(const c_string& path, FileMode mode,
@@ -509,7 +509,7 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
HdfsDriver driver
cdef cppclass HdfsPathInfo:
ObjectType kind;
ObjectType kind
c_string name
c_string owner
c_string group
@@ -563,21 +563,22 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil:
cdef extern from "arrow/io/memory.h" namespace "arrow::io" nogil:
cdef cppclass CBufferReader" arrow::io::BufferReader"\
(RandomAccessFile):
cdef cppclass CBufferReader\
" arrow::io::BufferReader"(RandomAccessFile):
CBufferReader(const shared_ptr[CBuffer]& buffer)
CBufferReader(const uint8_t* data, int64_t nbytes)
cdef cppclass CBufferOutputStream" arrow::io::BufferOutputStream"\
(OutputStream):
cdef cppclass CBufferOutputStream\
" arrow::io::BufferOutputStream"(OutputStream):
CBufferOutputStream(const shared_ptr[ResizableBuffer]& buffer)
cdef cppclass CMockOutputStream" arrow::io::MockOutputStream"\
(OutputStream):
cdef cppclass CMockOutputStream\
" arrow::io::MockOutputStream"(OutputStream):
CMockOutputStream()
int64_t GetExtentBytesWritten()
cdef cppclass CFixedSizeBufferWriter" arrow::io::FixedSizeBufferWriter"(WriteableFile):
cdef cppclass CFixedSizeBufferWriter\
" arrow::io::FixedSizeBufferWriter"(WriteableFile):
CFixedSizeBufferWriter(const shared_ptr[CBuffer]& buffer)
@@ -609,48 +610,45 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil:
c_string FormatMessageType(MessageType type)
cdef cppclass CMessageReader \
" arrow::ipc::MessageReader":
cdef cppclass CMessageReader" arrow::ipc::MessageReader":
CStatus ReadNextMessage(unique_ptr[CMessage]* out)
cdef cppclass CInputStreamMessageReader \
" arrow::ipc::InputStreamMessageReader":
cdef cppclass CInputStreamMessageReader\
" arrow::ipc::InputStreamMessageReader":
CInputStreamMessageReader(const shared_ptr[InputStream]& stream)
cdef cppclass CRecordBatchWriter \
" arrow::ipc::RecordBatchWriter":
cdef cppclass CRecordBatchWriter" arrow::ipc::RecordBatchWriter":
CStatus Close()
CStatus WriteRecordBatch(const CRecordBatch& batch)
cdef cppclass CRecordBatchReader \
" arrow::ipc::RecordBatchReader":
cdef cppclass CRecordBatchReader" arrow::ipc::RecordBatchReader":
shared_ptr[CSchema] schema()
CStatus ReadNextRecordBatch(shared_ptr[CRecordBatch]* batch)
cdef cppclass CRecordBatchStreamReader \
" arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader):
cdef cppclass CRecordBatchStreamReader\
" arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader):
@staticmethod
CStatus Open(const shared_ptr[InputStream]& stream,
shared_ptr[CRecordBatchStreamReader]* out)
@staticmethod
CStatus Open2" Open"(unique_ptr[CMessageReader] message_reader,
shared_ptr[CRecordBatchStreamReader]* out)
shared_ptr[CRecordBatchStreamReader]* out)
cdef cppclass CRecordBatchStreamWriter \
" arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter):
cdef cppclass CRecordBatchStreamWriter\
" arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter):
@staticmethod
CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
shared_ptr[CRecordBatchStreamWriter]* out)
cdef cppclass CRecordBatchFileWriter \
" arrow::ipc::RecordBatchFileWriter"(CRecordBatchWriter):
cdef cppclass CRecordBatchFileWriter\
" arrow::ipc::RecordBatchFileWriter"(CRecordBatchWriter):
@staticmethod
CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
shared_ptr[CRecordBatchFileWriter]* out)
cdef cppclass CRecordBatchFileReader \
" arrow::ipc::RecordBatchFileReader":
cdef cppclass CRecordBatchFileReader\
" arrow::ipc::RecordBatchFileReader":
@staticmethod
CStatus Open(const shared_ptr[RandomAccessFile]& file,
shared_ptr[CRecordBatchFileReader]* out)
@@ -724,7 +722,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
CStatus ConvertPySequence(object obj, CMemoryPool* pool,
shared_ptr[CArray]* out,
const shared_ptr[CDataType]& type,
int64_t size)
int64_t size)
CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type)
@@ -737,7 +735,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
shared_ptr[CChunkedArray]* out)
CStatus NdarrayToTensor(CMemoryPool* pool, object ao,
shared_ptr[CTensor]* out);
shared_ptr[CTensor]* out)
CStatus TensorToNdarray(const CTensor& tensor, object base,
PyObject** out)
python/pyarrow/io-hdfs.pxi
-1
View File
@@ -231,7 +231,6 @@ cdef class HadoopFileSystem:
check_status(self.client.get()
.GetPathInfo(c_path, info))
def ls(self, path, bint full_info):
cdef:
c_string c_path = tobytes(path)
python/pyarrow/io.pxi
+8 -2
View File
@@ -255,13 +255,18 @@ cdef class NativeFile:
if not hasattr(stream_or_path, 'read'):
stream = open(stream_or_path, 'wb')
cleanup = lambda: stream.close()
def cleanup():
stream.close()
else:
stream = stream_or_path
cleanup = lambda: None
def cleanup():
pass
done = False
exc_info = None
def bg_write():
try:
while not done or write_queue.qsize() > 0:
@@ -326,6 +331,7 @@ cdef class NativeFile:
done = False
exc_info = None
def bg_write():
try:
while not done or write_queue.qsize() > 0:
python/pyarrow/ipc.pxi
+1 -1
View File
@@ -279,7 +279,7 @@ cdef class _RecordBatchFileWriter(_RecordBatchWriter):
with nogil:
check_status(
CRecordBatchFileWriter.Open(self.sink.get(), schema.sp_schema,
&writer))
&writer))
# Cast to base class, because has same interface
self.writer = <shared_ptr[CRecordBatchWriter]> writer
python/pyarrow/lib.pxd
+1 -3
View File
@@ -17,6 +17,7 @@
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *
from pyarrow.includes.libarrow cimport CStatus
from cpython cimport PyObject
from libcpp cimport nullptr
@@ -24,9 +25,6 @@ cdef extern from "Python.h":
int PySlice_Check(object)
from pyarrow.includes.libarrow cimport CStatus
cdef int check_status(const CStatus& status) nogil except -1
python/pyarrow/lib.pyx
+9 -14
View File
@@ -19,31 +19,27 @@
# distutils: language = c++
# cython: embedsignature = True
import datetime
import decimal as _pydecimal
import multiprocessing
import numpy as np
import os
import six
from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
from cython.operator cimport dereference as deref
from pyarrow.includes.libarrow cimport *
from pyarrow.includes.common cimport PyObject_to_object
cimport pyarrow.includes.libarrow as libarrow
cimport cpython as cp
import datetime
import decimal as _pydecimal
import numpy as np
import six
from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
cdef _pandas():
import pandas as pd
return pd
arrow_init_numpy()
import numpy as np
set_numpy_nan(np.nan)
import multiprocessing
import os
cdef int CPU_COUNT = int(
os.environ.get('OMP_NUM_THREADS',
max(multiprocessing.cpu_count() // 2, 1)))
@@ -62,6 +58,7 @@ def cpu_count():
"""
return CPU_COUNT
def set_cpu_count(count):
global CPU_COUNT
CPU_COUNT = max(int(count), 1)
@@ -122,7 +119,5 @@ include "ipc.pxi"
# Feather format
include "feather.pxi"
#----------------------------------------------------------------------
# Public API
include "public-api.pxi"
python/pyarrow/plasma.pyx
+13 -11
View File
@@ -60,8 +60,8 @@ cdef extern from "plasma/common.h":
PLASMA_QUERY_LOCAL"plasma::PLASMA_QUERY_LOCAL",
PLASMA_QUERY_ANYWHERE"plasma::PLASMA_QUERY_ANYWHERE"
cdef int ObjectStatusLocal"plasma::ObjectStatusLocal";
cdef int ObjectStatusRemote"plasma::ObjectStatusRemote";
cdef int ObjectStatusLocal"plasma::ObjectStatusLocal"
cdef int ObjectStatusRemote"plasma::ObjectStatusRemote"
cdef extern from "plasma/client.h" nogil:
@@ -102,7 +102,7 @@ cdef extern from "plasma/client.h" nogil:
CStatus Wait(int64_t num_object_requests,
CObjectRequest* object_requests,
int num_ready_objects, int64_t timeout_ms,
int* num_objects_ready);
int* num_objects_ready)
CStatus Transfer(const char* addr, int port,
const CUniqueID& object_id)
@@ -312,9 +312,10 @@ cdef class PlasmaClient:
result = []
for i in range(object_buffers.size()):
if object_buffers[i].data_size != -1:
result.append(self._make_plasma_buffer(
object_ids[i], object_buffers[i].data,
object_buffers[i].data_size))
result.append(
self._make_plasma_buffer(object_ids[i],
object_buffers[i].data,
object_buffers[i].data_size))
else:
result.append(None)
return result
@@ -345,9 +346,10 @@ cdef class PlasmaClient:
self._get_object_buffers(object_ids, timeout_ms, &object_buffers)
result = []
for i in range(object_buffers.size()):
result.append(self._make_plasma_buffer(
object_ids[i], object_buffers[i].metadata,
object_buffers[i].metadata_size))
result.append(
self._make_plasma_buffer(object_ids[i],
object_buffers[i].metadata,
object_buffers[i].metadata_size))
return result
def seal(self, ObjectID object_id):
@@ -502,7 +504,7 @@ cdef class PlasmaClient:
object_requests.data(),
num_returns, timeout,
&num_objects_ready))
cdef int num_to_return = min(num_objects_ready, num_returns);
cdef int num_to_return = min(num_objects_ready, num_returns)
ready_ids = []
waiting_ids = set(object_ids)
cdef int num_returned = 0
@@ -510,7 +512,7 @@ cdef class PlasmaClient:
if num_returned == num_to_return:
break
if (object_requests[i].status == ObjectStatusLocal or
object_requests[i].status == ObjectStatusRemote):
object_requests[i].status == ObjectStatusRemote):
ready_ids.append(
ObjectID(object_requests[i].object_id.binary()))
waiting_ids.discard(
python/pyarrow/public-api.pxi
+6 -5
View File
@@ -47,7 +47,8 @@ cdef public api bint pyarrow_is_data_type(object type_):
return isinstance(type_, DataType)
cdef public api shared_ptr[CDataType] pyarrow_unwrap_data_type(object data_type):
cdef public api shared_ptr[CDataType] pyarrow_unwrap_data_type(
object data_type):
cdef DataType type_
if pyarrow_is_data_type(data_type):
type_ = <DataType>(data_type)
@@ -57,7 +58,7 @@ cdef public api shared_ptr[CDataType] pyarrow_unwrap_data_type(object data_type)
cdef public api object pyarrow_wrap_data_type(
const shared_ptr[CDataType]& type):
const shared_ptr[CDataType]& type):
cdef:
DataType out
@@ -149,7 +150,7 @@ cdef public api object pyarrow_wrap_array(const shared_ptr[CArray]& sp_array):
cdef public api object pyarrow_wrap_chunked_array(
const shared_ptr[CChunkedArray]& sp_array):
const shared_ptr[CChunkedArray]& sp_array):
if sp_array.get() == NULL:
raise ValueError('ChunkedArray was NULL')
@@ -177,7 +178,7 @@ cdef public api shared_ptr[CTensor] pyarrow_unwrap_tensor(object tensor):
cdef public api object pyarrow_wrap_tensor(
const shared_ptr[CTensor]& sp_tensor):
const shared_ptr[CTensor]& sp_tensor):
if sp_tensor.get() == NULL:
raise ValueError('Tensor was NULL')
@@ -238,7 +239,7 @@ cdef public api shared_ptr[CRecordBatch] pyarrow_unwrap_batch(object batch):
cdef public api object pyarrow_wrap_batch(
const shared_ptr[CRecordBatch]& cbatch):
const shared_ptr[CRecordBatch]& cbatch):
cdef RecordBatch batch = RecordBatch()
batch.init(cbatch)
return batch
python/pyarrow/scalar.pxi
+4 -4
View File
@@ -154,11 +154,11 @@ cdef class Time32Value(ArrayValue):
CTime32Type* dtype = <CTime32Type*> ap.type().get()
if dtype.unit() == TimeUnit_SECOND:
return (datetime.datetime(1970, 1, 1) +
datetime.timedelta(seconds=ap.Value(self.index))).time()
delta = datetime.timedelta(seconds=ap.Value(self.index))
return (datetime.datetime(1970, 1, 1) + delta).time()
else:
return (datetime.datetime(1970, 1, 1) +
datetime.timedelta(milliseconds=ap.Value(self.index))).time()
delta = datetime.timedelta(milliseconds=ap.Value(self.index))
return (datetime.datetime(1970, 1, 1) + delta).time()
cdef class Time64Value(ArrayValue):
python/pyarrow/table.pxi
+2 -3
View File
@@ -495,7 +495,6 @@ cdef class RecordBatch:
entries.append((name, column))
return OrderedDict(entries)
def to_pandas(self, nthreads=None):
"""
Convert the arrow::RecordBatch to a pandas DataFrame
@@ -585,7 +584,6 @@ def table_to_blocks(Table table, int nthreads):
return PyObject_to_object(result_obj)
cdef class Table:
"""
A collection of top-level named, equal length Arrow arrays.
@@ -897,7 +895,8 @@ cdef class Table:
"""
Number of rows in this table.
Due to the definition of a table, all columns have the same number of rows.
Due to the definition of a table, all columns have the same number of
rows.
Returns
-------
python/pyarrow/types.pxi
+1 -1
View File
@@ -419,7 +419,7 @@ cdef DataType primitive_type(Type type):
_type_cache[type] = out
return out
#------------------------------------------------------------
# -----------------------------------------------------------
# Type factory functions
cdef int convert_metadata(dict metadata,