Fixed case for Integer specialization without nulls

python/pyarrow/tests/test_table.py

@@ -51,20 +51,25 @@ def test_recordbatch_from_to_pandas():
 
 
 def test_recordbatchlist_to_pandas():
-    # NOTE - dtype='uint32' is converted to float64 when not zero copy
-    data = pd.DataFrame({
-        'c1': np.array([1.0, 2.0, 3.0, 4.0, 5.0], dtype='float64'),
-        'c2': ['foo', 'bar', None, 'baz', 'qux']
+    data1 = pd.DataFrame({
+        'c1': np.array([1, 1, 2], dtype='uint32'),
+        'c2': np.array([1.0, 2.0, 3.0], dtype='float64'),
+        'c3': [True, None, False],
+        'c4': ['foo', 'bar', None]
     })
 
-    split = int(len(data) / 2)
-    data1 = data[:split]
-    data2 = data[split:]
+    data2 = pd.DataFrame({
+        'c1': np.array([3, 5], dtype='uint32'),
+        'c2': np.array([4.0, 5.0], dtype='float64'),
+        'c3': [True, True],
+        'c4': ['baz', 'qux']
+    })
 
     batch1 = pa.RecordBatch.from_pandas(data1)
     batch2 = pa.RecordBatch.from_pandas(data2)
 
     result = pa.RecordBatchList.to_pandas([batch1, batch2])
+    data = pd.concat([data1, data2], ignore_index=True)
     assert_frame_equal(data, result)
 
 

python/src/pyarrow/adapters/pandas.cc

@@ -630,7 +630,7 @@ class ArrowDeserializer {
 
     if (out_ == NULL) {
       // Error occurred, trust that SimpleNew set the error state
-      return Status::OK();//Invalid("Error in PyArray_SimpleNewFromData");
+      return Status::OK();
     }
 
     set_numpy_metadata(type, col_->type().get(), out_);
@@ -670,10 +670,9 @@ class ArrowDeserializer {
     arrow_traits<T2>::is_pandas_numeric_nullable, Status>::type
   ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
     typedef typename arrow_traits<T2>::T T;
-    bool has_null_values = data->null_count() > 0;
     size_t chunk_offset = 0;
 
-    if (data->num_chunks() == 1 && !has_null_values) {
+    if (data->num_chunks() == 1 && data->null_count() == 0) {
       return ConvertValuesZeroCopy<TYPE>(data->chunk(0));
     }
 
@@ -686,15 +685,12 @@ class ArrowDeserializer {
       const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
       T* out_values = reinterpret_cast<T*>(PyArray_DATA(out_)) + chunk_offset;
 
-      if (has_null_values) {
+      if (arr->null_count() > 0) {
         for (int64_t i = 0; i < arr->length(); ++i) {
           out_values[i] = arr->IsNull(i) ? arrow_traits<T2>::na_value : in_values[i];
         }
       } else {
-        // TODO - copy entire chunck?
-        for (int64_t i = 0; i < arr->length(); ++i) {
-          out_values[i] = in_values[i];
-        }
+        memcpy(out_values, in_values, sizeof(T) * arr->length());
       }
 
       chunk_offset += arr->length();
@@ -709,35 +705,43 @@ class ArrowDeserializer {
     arrow_traits<T2>::is_pandas_numeric_not_nullable, Status>::type
   ConvertValues(const std::shared_ptr<arrow::ChunkedArray>& data) {
     typedef typename arrow_traits<T2>::T T;
-    bool has_null_values = data->null_count() > 0;
     size_t chunk_offset = 0;
 
-    if (data->num_chunks() == 1 && !has_null_values) {
+    if (data->num_chunks() == 1 && data->null_count() == 0) {
       return ConvertValuesZeroCopy<TYPE>(data->chunk(0));
     }
 
-    // TODO - why cast to double?
-    RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
+    if (data->null_count() > 0) {
+      RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64));
 
-    for (int c = 0; c < data->num_chunks(); c++) {
-      const std::shared_ptr<Array> arr = data->chunk(c);
-      arrow::PrimitiveArray* prim_arr = static_cast<arrow::PrimitiveArray*>(
-          arr.get());
-      const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
-      // Upcast to double, set NaN as appropriate
-      double* out_values = reinterpret_cast<double*>(PyArray_DATA(out_)) + chunk_offset;
+      for (int c = 0; c < data->num_chunks(); c++) {
+        const std::shared_ptr<Array> arr = data->chunk(c);
+        arrow::PrimitiveArray* prim_arr = static_cast<arrow::PrimitiveArray*>(
+            arr.get());
+        const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+        // Upcast to double, set NaN as appropriate
+        double* out_values = reinterpret_cast<double*>(PyArray_DATA(out_)) + chunk_offset;
 
-      if (arr->null_count() > 0) {
         for (int i = 0; i < arr->length(); ++i) {
           out_values[i] = prim_arr->IsNull(i) ? NAN : in_values[i];
         }
-      } else {
-        for (int i = 0; i < arr->length(); ++i) {
-          out_values[i] = in_values[i];
-        }
-      }
 
-      chunk_offset += arr->length();
+        chunk_offset += arr->length();
+      }
+    } else {
+      RETURN_NOT_OK(AllocateOutput(arrow_traits<TYPE>::npy_type));
+
+      for (int c = 0; c < data->num_chunks(); c++) {
+        const std::shared_ptr<Array> arr = data->chunk(c);
+        arrow::PrimitiveArray* prim_arr = static_cast<arrow::PrimitiveArray*>(
+            arr.get());
+        const T* in_values = reinterpret_cast<const T*>(prim_arr->data()->data());
+        T* out_values = reinterpret_cast<T*>(PyArray_DATA(out_)) + chunk_offset;
+
+        memcpy(out_values, in_values, sizeof(T) * arr->length());
+
+        chunk_offset += arr->length();
+      }
     }
 
     return Status::OK();