Update documentation and testing.

This commit is contained in:
Robert Nishihara authored and Wes McKinney committed 2018-02-19 17:58:09 -05:00
1 parent b796fbdb6c
commit 0fc9ecfddb
5 files changed
+5 -35

No files matched your search

+3 -4
View File
@@ -317,9 +317,8 @@ An object can be reconstructed from its component-based representation using
Serializing pandas Objects
--------------------------
We provide a serialization context that has optimized handling of pandas
objects like ``DataFrame`` and ``Series``. This can be created with
``pyarrow.pandas_serialization_context()``. Combined with component-based
The default serialization context has optimized handling of pandas
objects like ``DataFrame`` and ``Series``. Combined with component-based
serialization above, this enables zero-copy transport of pandas DataFrame
objects not containing any Python objects:
@@ -327,7 +326,7 @@ objects not containing any Python objects:
import pandas as pd
df = pd.DataFrame({'a': [1, 2, 3, 4, 5]})
context = pa.pandas_serialization_context()
context = pa.default_serialization_context()
serialized_df = context.serialize(df)
df_components = serialized_df.to_components()
original_df = context.deserialize_components(df_components)
-1
View File
@@ -125,7 +125,6 @@ from pyarrow.ipc import (Message, MessageReader,
localfs = LocalFileSystem.get_instance()
from pyarrow.serialization import (default_serialization_context,
pandas_serialization_context,
register_default_serialization_handlers,
register_torch_serialization_handlers)
+1 -27
View File
@@ -22,8 +22,7 @@ import sys
import numpy as np
from pyarrow.compat import builtin_pickle
from pyarrow.lib import (SerializationContext, _default_serialization_context,
frombuffer)
from pyarrow.lib import SerializationContext, _default_serialization_context
try:
import cloudpickle
@@ -44,23 +43,6 @@ def _deserialize_numpy_array_list(data):
return np.array(data[0], dtype=np.dtype(data[1]))
def _pickle_to_buffer(x):
pickled = builtin_pickle.dumps(x, protocol=builtin_pickle.HIGHEST_PROTOCOL)
return frombuffer(pickled)
def _load_pickle_from_buffer(data):
as_memoryview = memoryview(data)
if six.PY2:
return builtin_pickle.loads(as_memoryview.tobytes())
else:
return builtin_pickle.loads(as_memoryview)
_serialize_numpy_array_pickle = _pickle_to_buffer
_deserialize_numpy_array_pickle = _load_pickle_from_buffer
# ----------------------------------------------------------------------
# pandas-specific serialization matters
@@ -190,11 +172,3 @@ def default_serialization_context():
register_default_serialization_handlers(_default_serialization_context)
def pandas_serialization_context():
context = default_serialization_context()
context.register_type(np.ndarray, 'np.array',
custom_serializer=_serialize_numpy_array_pickle,
custom_deserializer=_deserialize_numpy_array_pickle)
return context
+1 -1
View File
@@ -1613,7 +1613,7 @@ def _fully_loaded_dataframe_example():
def _check_serialize_components_roundtrip(df):
ctx = pa.pandas_serialization_context()
ctx = pa.default_serialization_context()
components = ctx.serialize(df).to_components()
deserialized = ctx.deserialize_components(components)
@@ -284,8 +284,6 @@ def test_clone():
def test_primitive_serialization(large_buffer):
for obj in PRIMITIVE_OBJECTS:
serialization_roundtrip(obj, large_buffer)
serialization_roundtrip(obj, large_buffer,
pa.pandas_serialization_context())
def test_serialize_to_buffer():