Draft implementations of Arrow file read/write from Python

python/CMakeLists.txt

@@ -352,6 +352,8 @@ ADD_THIRDPARTY_LIB(arrow
   SHARED_LIB ${ARROW_SHARED_LIB})
 ADD_THIRDPARTY_LIB(arrow_io
   SHARED_LIB ${ARROW_IO_SHARED_LIB})
+ADD_THIRDPARTY_LIB(arrow_ipc
+  SHARED_LIB ${ARROW_IPC_SHARED_LIB})
 
 ############################################################
 # Linker setup
@@ -423,6 +425,7 @@ set(CYTHON_EXTENSIONS
   config
   error
   io
+  ipc
   scalar
   schema
   table
@@ -442,6 +445,7 @@ set(PYARROW_SRCS
 set(LINK_LIBS
   arrow
   arrow_io
+  arrow_ipc
 )
 
 if(PARQUET_FOUND AND PARQUET_ARROW_FOUND)

python/cmake_modules/FindArrow.cmake

@@ -47,10 +47,16 @@ find_library(ARROW_IO_LIB_PATH NAMES arrow_io
   ${ARROW_SEARCH_LIB_PATH}
   NO_DEFAULT_PATH)
 
+find_library(ARROW_IPC_LIB_PATH NAMES arrow_ipc
+  PATHS
+  ${ARROW_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH)
+
 if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
   set(ARROW_FOUND TRUE)
   set(ARROW_LIB_NAME libarrow)
   set(ARROW_IO_LIB_NAME libarrow_io)
+  set(ARROW_IPC_LIB_NAME libarrow_ipc)
 
   set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH})
   set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a)
@@ -58,9 +64,14 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH)
 
   set(ARROW_IO_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IO_LIB_NAME}.a)
   set(ARROW_IO_SHARED_LIB ${ARROW_LIBS}/${ARROW_IO_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
+  set(ARROW_IPC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IPC_LIB_NAME}.a)
+  set(ARROW_IPC_SHARED_LIB ${ARROW_LIBS}/${ARROW_IPC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
   if (NOT Arrow_FIND_QUIETLY)
     message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
     message(STATUS "Found the Arrow IO library: ${ARROW_IO_LIB_PATH}")
+    message(STATUS "Found the Arrow IPC library: ${ARROW_IPC_LIB_PATH}")
   endif ()
 else ()
   if (NOT Arrow_FIND_QUIETLY)

python/pyarrow/__init__.py

@@ -41,5 +41,4 @@ from pyarrow.schema import (null, bool_,
                             list_, struct, field,
                             DataType, Field, Schema, schema)
 
-from pyarrow.array import RowBatch
-from pyarrow.table import Column, Table, from_pandas_dataframe
+from pyarrow.table import Column, RecordBatch, Table, from_pandas_dataframe

python/pyarrow/array.pyx

@@ -262,35 +262,3 @@ cdef object series_as_ndarray(object obj):
         result = obj
 
     return result
-
-#----------------------------------------------------------------------
-# Table-like data structures
-
-cdef class RowBatch:
-    """
-
-    """
-    cdef readonly:
-        Schema schema
-        int num_rows
-        list arrays
-
-    def __cinit__(self, Schema schema, int num_rows, list arrays):
-        self.schema = schema
-        self.num_rows = num_rows
-        self.arrays = arrays
-
-        if len(self.schema) != len(arrays):
-            raise ValueError('Mismatch number of data arrays and '
-                             'schema fields')
-
-    def __len__(self):
-        return self.num_rows
-
-    property num_columns:
-
-        def __get__(self):
-            return len(self.arrays)
-
-    def __getitem__(self, i):
-        return self.arrays[i]

python/pyarrow/includes/libarrow.pxd

@@ -149,6 +149,19 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
         const shared_ptr[CDataType]& type()
         const shared_ptr[CChunkedArray]& data()
 
+    cdef cppclass CRecordBatch" arrow::RecordBatch":
+        CRecordBatch(const shared_ptr[CSchema]& schema, int32_t num_rows,
+                     const vector[shared_ptr[CArray]]& columns)
+
+        const shared_ptr[CSchema]& schema()
+        const shared_ptr[CArray]& column(int i)
+        const c_string& column_name(int i)
+
+        const vector[shared_ptr[CArray]]& columns()
+
+        int num_columns()
+        int32_t num_rows()
+
     cdef cppclass CTable" arrow::Table":
         CTable(const c_string& name, const shared_ptr[CSchema]& schema,
                const vector[shared_ptr[CColumn]]& columns)

python/pyarrow/includes/libarrow_ipc.pxd

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# distutils: language = c++
+
+from pyarrow.includes.common cimport *
+from pyarrow.includes.libarrow cimport (MemoryPool, CArray, CSchema,
+                                        CRecordBatch)
+from pyarrow.includes.libarrow_io cimport (OutputStream, ReadableFileInterface)
+
+cdef extern from "arrow/ipc/file.h" namespace "arrow::ipc" nogil:
+
+    cdef cppclass CFileWriter " arrow::ipc::FileWriter":
+        @staticmethod
+        CStatus Open(OutputStream* sink, const shared_ptr[CSchema]& schema,
+                     shared_ptr[CFileWriter]* out)
+
+        CStatus WriteRecordBatch(const vector[shared_ptr[CArray]]& columns,
+                                 int32_t num_rows)
+
+        CStatus Close()
+
+    cdef cppclass CFileReader " arrow::ipc::FileReader":
+
+        @staticmethod
+        CStatus Open(const shared_ptr[ReadableFileInterface]& file,
+                     shared_ptr[CFileReader]* out)
+
+        @staticmethod
+        CStatus Open2" Open"(const shared_ptr[ReadableFileInterface]& file,
+                     int64_t footer_offset, shared_ptr[CFileReader]* out)
+
+        const shared_ptr[CSchema]& schema()
+
+        int num_dictionaries()
+        int num_record_batches()
+
+        CStatus GetRecordBatch(int i, shared_ptr[CRecordBatch]* batch)

python/pyarrow/io.pxd

@@ -29,6 +29,7 @@ cdef class NativeFile:
         shared_ptr[OutputStream] wr_file
         bint is_readonly
         bint is_open
+        bint own_file
 
     # By implementing these "virtual" functions (all functions in Cython
     # extension classes are technically virtual in the C++ sense) we can expose

python/pyarrow/io.pyx

@@ -274,6 +274,7 @@ cdef class HdfsClient:
         out.buffer_size = c_buffer_size
         out.parent = self
         out.is_open = True
+        out.own_file = True
 
         return out
 
@@ -326,9 +327,10 @@ cdef class NativeFile:
 
     def __cinit__(self):
         self.is_open = False
+        self.own_file = False
 
     def __dealloc__(self):
-        if self.is_open:
+        if self.is_open and self.own_file:
             self.close()
 
     def __enter__(self):

python/pyarrow/ipc.pyx

@@ -0,0 +1,155 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cython wrappers for arrow::ipc
+
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+
+from pyarrow.includes.libarrow cimport *
+from pyarrow.includes.libarrow_io cimport *
+from pyarrow.includes.libarrow_ipc cimport *
+cimport pyarrow.includes.pyarrow as pyarrow
+
+from pyarrow.error cimport check_cstatus
+from pyarrow.io cimport NativeFile
+from pyarrow.schema cimport Schema
+from pyarrow.table cimport RecordBatch
+
+from pyarrow.compat import frombytes, tobytes
+import pyarrow.io as io
+
+cimport cpython as cp
+
+
+cdef get_reader(source, shared_ptr[ReadableFileInterface]* reader):
+    cdef NativeFile nf
+
+    if isinstance(source, bytes):
+        source = io.BytesReader(source)
+    elif not isinstance(source, io.NativeFile) and hasattr(source, 'read'):
+        # Optimistically hope this is file-like
+        source = io.PythonFileInterface(source, mode='r')
+
+    if isinstance(source, NativeFile):
+        nf = source
+
+        # TODO: what about read-write sources (e.g. memory maps)
+        if not nf.is_readonly:
+            raise IOError('Native file is not readable')
+
+        nf.read_handle(reader)
+    else:
+        raise TypeError('Unable to read from object of type: {0}'
+                        .format(type(source)))
+
+
+cdef get_writer(source, shared_ptr[OutputStream]* writer):
+    cdef NativeFile nf
+
+    if not isinstance(source, io.NativeFile) and hasattr(source, 'write'):
+        # Optimistically hope this is file-like
+        source = io.PythonFileInterface(source, mode='w')
+
+    if isinstance(source, io.NativeFile):
+        nf = source
+
+        if nf.is_readonly:
+            raise IOError('Native file is not writeable')
+
+        nf.write_handle(writer)
+    else:
+        raise TypeError('Unable to read from object of type: {0}'
+                        .format(type(source)))
+
+
+cdef class ArrowFileWriter:
+    cdef:
+        shared_ptr[CFileWriter] writer
+        shared_ptr[OutputStream] sink
+        bint closed
+
+    def __cinit__(self, sink, Schema schema):
+        self.closed = True
+        get_writer(sink, &self.sink)
+
+        with nogil:
+            check_cstatus(CFileWriter.Open(self.sink.get(), schema.sp_schema,
+                                           &self.writer))
+
+        self.closed = False
+
+    def __dealloc__(self):
+        if not self.closed:
+            self.close()
+
+    def write_record_batch(self, RecordBatch batch):
+        cdef CRecordBatch* bptr = batch.batch
+        with nogil:
+            check_cstatus(self.writer.get()
+                          .WriteRecordBatch(bptr.columns(), bptr.num_rows()))
+
+    def close(self):
+        with nogil:
+            check_cstatus(self.writer.get().Close())
+        self.closed = True
+
+
+cdef class ArrowFileReader:
+    cdef:
+        shared_ptr[CFileReader] reader
+
+    def __cinit__(self, source, footer_offset=None):
+        cdef shared_ptr[ReadableFileInterface] reader
+        get_reader(source, &reader)
+
+        cdef int64_t offset = 0
+        if footer_offset is not None:
+            offset = footer_offset
+
+        with nogil:
+            if offset != 0:
+                check_cstatus(CFileReader.Open2(reader, offset, &self.reader))
+            else:
+                check_cstatus(CFileReader.Open(reader, &self.reader))
+
+    property num_dictionaries:
+
+        def __get__(self):
+            return self.reader.get().num_dictionaries()
+
+    property num_record_batches:
+
+        def __get__(self):
+            return self.reader.get().num_record_batches()
+
+    def get_record_batch(self, int i):
+        cdef:
+            shared_ptr[CRecordBatch] batch
+            RecordBatch result
+
+        if i < 0 or i >= self.num_record_batches:
+            raise ValueError('Batch number {0} out of range'.format(i))
+
+        with nogil:
+            check_cstatus(self.reader.get().GetRecordBatch(i, &batch))
+
+        result = RecordBatch()
+        result.init(batch)
+
+        return result

python/pyarrow/table.pxd

@@ -16,7 +16,10 @@
 # under the License.
 
 from pyarrow.includes.common cimport shared_ptr
-from pyarrow.includes.libarrow cimport CChunkedArray, CColumn, CTable
+from pyarrow.includes.libarrow cimport (CChunkedArray, CColumn, CTable,
+                                        CRecordBatch)
+
+from pyarrow.schema cimport Schema
 
 
 cdef class ChunkedArray:
@@ -41,6 +44,16 @@ cdef class Table:
     cdef:
         shared_ptr[CTable] sp_table
         CTable* table
-    
+
     cdef init(self, const shared_ptr[CTable]& table)
     cdef _check_nullptr(self)
+
+
+cdef class RecordBatch:
+    cdef:
+        shared_ptr[CRecordBatch] sp_batch
+        CRecordBatch* batch
+        Schema _schema
+
+    cdef init(self, const shared_ptr[CRecordBatch]& table)
+    cdef _check_nullptr(self)

python/pyarrow/table.pyx

@@ -144,6 +144,124 @@ cdef class Column:
             return chunked_array
 
 
+cdef _schema_from_arrays(arrays, names, shared_ptr[CSchema]* schema):
+    cdef:
+        Array arr
+        c_string c_name
+        vector[shared_ptr[CField]] fields
+
+    cdef int K = len(arrays)
+
+    fields.resize(K)
+    for i in range(K):
+        arr = arrays[i]
+        c_name = tobytes(names[i])
+        fields[i].reset(new CField(c_name, arr.type.sp_type, True))
+
+    schema.reset(new CSchema(fields))
+
+
+
+cdef _dataframe_to_arrays(df, name, timestamps_to_ms):
+    from pyarrow.array import from_pandas_series
+
+    cdef:
+        list names = []
+        list arrays = []
+
+    for name in df.columns:
+        col = df[name]
+        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms)
+
+        names.append(name)
+        arrays.append(arr)
+
+    return names, arrays
+
+
+cdef class RecordBatch:
+
+    def __cinit__(self):
+        self.batch = NULL
+        self._schema = None
+
+    cdef init(self, const shared_ptr[CRecordBatch]& batch):
+        self.sp_batch = batch
+        self.batch = batch.get()
+
+    cdef _check_nullptr(self):
+        if self.batch == NULL:
+            raise ReferenceError("Object not initialized")
+
+    def __len__(self):
+        self._check_nullptr()
+        return self.batch.num_rows()
+
+    property num_columns:
+
+        def __get__(self):
+            self._check_nullptr()
+            return self.batch.num_columns()
+
+    property num_rows:
+
+        def __get__(self):
+            return len(self)
+
+    property schema:
+
+        def __get__(self):
+            cdef Schema schema
+            self._check_nullptr()
+            if self._schema is None:
+                schema = Schema()
+                schema.init_schema(self.batch.schema())
+                self._schema = schema
+
+            return self._schema
+
+    def __getitem__(self, i):
+        cdef Array arr = Array()
+        arr.init(self.batch.column(i))
+        return arr
+
+    @classmethod
+    def from_pandas(cls, df):
+        """
+        Convert pandas.DataFrame to an Arrow RecordBatch
+        """
+        names, arrays = _dataframe_to_arrays(df, None, False)
+        return cls.from_arrays(names, arrays)
+
+    @staticmethod
+    def from_arrays(names, arrays):
+        cdef:
+            Array arr
+            RecordBatch result
+            c_string c_name
+            shared_ptr[CSchema] schema
+            shared_ptr[CRecordBatch] batch
+            vector[shared_ptr[CArray]] c_arrays
+            int32_t num_rows
+
+        if len(arrays) == 0:
+            raise ValueError('Record batch cannot contain no arrays (for now)')
+
+        num_rows = len(arrays[0])
+        _schema_from_arrays(arrays, names, &schema)
+
+        for i in range(len(arrays)):
+            arr = arrays[i]
+            c_arrays.push_back(arr.sp_array)
+
+        batch.reset(new CRecordBatch(schema, num_rows, c_arrays))
+
+        result = RecordBatch()
+        result.init(batch)
+
+        return result
+
+
 cdef class Table:
     '''
     Do not call this class's constructor directly.
@@ -161,38 +279,50 @@ cdef class Table:
             raise ReferenceError("Table object references a NULL pointer."
                     "Not initialized.")
 
-    @staticmethod
-    def from_pandas(df, name=None):
-        return from_pandas_dataframe(df, name=name)
+    @classmethod
+    def from_pandas(cls, df, name=None, timestamps_to_ms=False):
+        """
+        Convert pandas.DataFrame to an Arrow Table
+
+        Parameters
+        ----------
+        df: pandas.DataFrame
+
+        name: str
+
+        timestamps_to_ms: bool
+            Convert datetime columns to ms resolution. This is needed for
+            compability with other functionality like Parquet I/O which
+            only supports milliseconds.
+        """
+        names, arrays = _dataframe_to_arrays(df, name=name,
+                                             timestamps_to_ms=timestamps_to_ms)
+        return cls.from_arrays(names, arrays, name=name)
 
     @staticmethod
     def from_arrays(names, arrays, name=None):
         cdef:
             Array arr
-            Table result
             c_string c_name
             vector[shared_ptr[CField]] fields
             vector[shared_ptr[CColumn]] columns
+            Table result
             shared_ptr[CSchema] schema
             shared_ptr[CTable] table
 
-        cdef int K = len(arrays)
+        _schema_from_arrays(arrays, names, &schema)
 
-        fields.resize(K)
+        cdef int K = len(arrays)
         columns.resize(K)
         for i in range(K):
             arr = arrays[i]
-            c_name = tobytes(names[i])
-
-            fields[i].reset(new CField(c_name, arr.type.sp_type, True))
-            columns[i].reset(new CColumn(fields[i], arr.sp_array))
+            columns[i].reset(new CColumn(schema.get().field(i), arr.sp_array))
 
         if name is None:
             c_name = ''
         else:
             c_name = tobytes(name)
 
-        schema.reset(new CSchema(fields))
         table.reset(new CTable(c_name, schema, columns))
 
         result = Table()
@@ -268,32 +398,4 @@ cdef class Table:
 
 
 
-def from_pandas_dataframe(object df, name=None, timestamps_to_ms=False):
-    """
-    Convert pandas.DataFrame to an Arrow Table
-
-    Parameters
-    ----------
-    df: pandas.DataFrame
-
-    name: str
-
-    timestamps_to_ms: bool
-        Convert datetime columns to ms resolution. This is needed for
-        compability with other functionality like Parquet I/O which
-        only supports milliseconds.
-    """
-    from pyarrow.array import from_pandas_series
-
-    cdef:
-        list names = []
-        list arrays = []
-
-    for name in df.columns:
-        col = df[name]
-        arr = from_pandas_series(col, timestamps_to_ms=timestamps_to_ms)
-
-        names.append(name)
-        arrays.append(arr)
-
-    return Table.from_arrays(names, arrays, name=name)
+from_pandas_dataframe = Table.from_pandas

python/pyarrow/tests/test_ipc.py

@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import io
+
+from pandas.util.testing import assert_frame_equal
+import numpy as np
+import pandas as pd
+
+import pyarrow as A
+import pyarrow.ipc as ipc
+
+
+def test_ipc_simple_roundtrip():
+    nrows = 5
+    df = pd.DataFrame({
+        'one': np.random.randn(nrows),
+        'two': ['foo', np.nan, 'bar', 'bazbaz', 'qux']})
+
+    batch = A.RecordBatch.from_pandas(df)
+
+    sink = io.BytesIO()
+    writer = ipc.ArrowFileWriter(sink, batch.schema)
+
+    num_batches = 5
+    frames = []
+    for i in range(num_batches):
+        unique_df = df.copy()
+        unique_df['one'] = np.random.randn(nrows)
+
+        batch = A.RecordBatch.from_pandas(unique_df)
+        writer.write_record_batch(batch)
+        frames.append(unique_df)
+
+    writer.close()
+
+    file_contents = sink.getvalue()
+    reader = ipc.ArrowFileReader(file_contents)
+
+    assert reader.num_record_batches == num_batches
+
+    for i in range(num_batches):
+        # it works. Must convert back to DataFrame
+        reader.get_record_batch(i)
+
+
+def big_batch():
+    df = pd.DataFrame(
+        np.random.randn(2**4, 2**23).T,
+        columns=[str(i) for i in range(2**4)]
+    )
+    return A.RecordBatch.from_pandas(df)
+
+
+def write_to_memory(batch):
+    sink = io.BytesIO()
+    write_file(batch, sink)
+    return sink.getvalue()
+
+
+def write_file(batch, sink):
+    writer = ipc.ArrowFileWriter(sink, batch.schema)
+    writer.write_record_batch(batch)
+    writer.close()
+
+
+def read_file(source):
+    reader = ipc.ArrowFileReader(source)
+    return [reader.get_record_batch(i)
+            for i in range(reader.num_record_batches)]

python/pyarrow/tests/test_table.py

@@ -15,60 +15,52 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import unittest
 import pyarrow as A
 
 
-class TestRowBatch(unittest.TestCase):
+def test_recordbatch_basics():
+    data = [
+        A.from_pylist(range(5)),
+        A.from_pylist([-10, -5, 0, 5, 10])
+    ]
 
-    def test_basics(self):
-        data = [
-            A.from_pylist(range(5)),
-            A.from_pylist([-10, -5, 0, 5, 10])
-        ]
-        num_rows = 5
+    batch = A.RecordBatch.from_arrays(['c0', 'c1'], data)
 
-        descr = A.schema([A.field('c0', data[0].type),
-                          A.field('c1', data[1].type)])
-
-        batch = A.RowBatch(descr, num_rows, data)
-
-        assert len(batch) == num_rows
-        assert batch.num_rows == num_rows
-        assert batch.num_columns == len(data)
+    assert len(batch) == 5
+    assert batch.num_rows == 5
+    assert batch.num_columns == len(data)
 
 
-class TestTable(unittest.TestCase):
+def test_table_basics():
+    data = [
+        A.from_pylist(range(5)),
+        A.from_pylist([-10, -5, 0, 5, 10])
+    ]
+    table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
+    assert table.name == 'table_name'
+    assert len(table) == 5
+    assert table.num_rows == 5
+    assert table.num_columns == 2
+    assert table.shape == (5, 2)
 
-    def test_basics(self):
-        data = [
-            A.from_pylist(range(5)),
-            A.from_pylist([-10, -5, 0, 5, 10])
-        ]
-        table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
-        assert table.name == 'table_name'
-        assert len(table) == 5
-        assert table.num_rows == 5
-        assert table.num_columns == 2
-        assert table.shape == (5, 2)
+    for col in table.itercolumns():
+        for chunk in col.data.iterchunks():
+            assert chunk is not None
 
-        for col in table.itercolumns():
-            for chunk in col.data.iterchunks():
-                assert chunk is not None
 
-    def test_pandas(self):
-        data = [
-            A.from_pylist(range(5)),
-            A.from_pylist([-10, -5, 0, 5, 10])
-        ]
-        table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
+def test_table_pandas():
+    data = [
+        A.from_pylist(range(5)),
+        A.from_pylist([-10, -5, 0, 5, 10])
+    ]
+    table = A.Table.from_arrays(('a', 'b'), data, 'table_name')
 
-        # TODO: Use this part once from_pandas is implemented
-        # data = {'a': range(5), 'b': [-10, -5, 0, 5, 10]}
-        # df = pd.DataFrame(data)
-        # A.Table.from_pandas(df)
+    # TODO: Use this part once from_pandas is implemented
+    # data = {'a': range(5), 'b': [-10, -5, 0, 5, 10]}
+    # df = pd.DataFrame(data)
+    # A.Table.from_pandas(df)
 
-        df = table.to_pandas()
-        assert set(df.columns) == set(('a', 'b'))
-        assert df.shape == (5, 2)
-        assert df.ix[0, 'b'] == -10
+    df = table.to_pandas()
+    assert set(df.columns) == set(('a', 'b'))
+    assert df.shape == (5, 2)
+    assert df.loc[0, 'b'] == -10

python/setup.py

@@ -102,6 +102,7 @@ class build_ext(_build_ext):
         'config',
         'error',
         'io',
+        'ipc',
         'parquet',
         'scalar',
         'schema',

python/src/pyarrow/io.cc

@@ -47,9 +47,9 @@ static arrow::Status CheckPyError() {
     PyErr_Fetch(&exc_type, &exc_value, &traceback);
     PyObjectStringify stringified(exc_value);
     std::string message(stringified.bytes);
-    Py_DECREF(exc_type);
-    Py_DECREF(exc_value);
-    Py_DECREF(traceback);
+    Py_XDECREF(exc_type);
+    Py_XDECREF(exc_value);
+    Py_XDECREF(traceback);
     PyErr_Clear();
     return arrow::Status::IOError(message);
   }