mirror of
https://github.com/apache/arrow.git
synced 2026-10-29 00:56:31 +08:00
Update documentation and testing.
This commit is contained in:
1 parent
b796fbdb6c
commit
0fc9ecfddb
5 files changed
+5
-35
No files matched your search
@@ -317,9 +317,8 @@ An object can be reconstructed from its component-based representation using
|
||||
Serializing pandas Objects
|
||||
--------------------------
|
||||
|
||||
We provide a serialization context that has optimized handling of pandas
|
||||
objects like ``DataFrame`` and ``Series``. This can be created with
|
||||
``pyarrow.pandas_serialization_context()``. Combined with component-based
|
||||
The default serialization context has optimized handling of pandas
|
||||
objects like ``DataFrame`` and ``Series``. Combined with component-based
|
||||
serialization above, this enables zero-copy transport of pandas DataFrame
|
||||
objects not containing any Python objects:
|
||||
|
||||
@@ -327,7 +326,7 @@ objects not containing any Python objects:
|
||||
|
||||
import pandas as pd
|
||||
df = pd.DataFrame({'a': [1, 2, 3, 4, 5]})
|
||||
context = pa.pandas_serialization_context()
|
||||
context = pa.default_serialization_context()
|
||||
serialized_df = context.serialize(df)
|
||||
df_components = serialized_df.to_components()
|
||||
original_df = context.deserialize_components(df_components)
|
||||
|
||||
@@ -125,7 +125,6 @@ from pyarrow.ipc import (Message, MessageReader,
|
||||
localfs = LocalFileSystem.get_instance()
|
||||
|
||||
from pyarrow.serialization import (default_serialization_context,
|
||||
pandas_serialization_context,
|
||||
register_default_serialization_handlers,
|
||||
register_torch_serialization_handlers)
|
||||
|
||||
|
||||
@@ -22,8 +22,7 @@ import sys
|
||||
import numpy as np
|
||||
|
||||
from pyarrow.compat import builtin_pickle
|
||||
from pyarrow.lib import (SerializationContext, _default_serialization_context,
|
||||
frombuffer)
|
||||
from pyarrow.lib import SerializationContext, _default_serialization_context
|
||||
|
||||
try:
|
||||
import cloudpickle
|
||||
@@ -44,23 +43,6 @@ def _deserialize_numpy_array_list(data):
|
||||
return np.array(data[0], dtype=np.dtype(data[1]))
|
||||
|
||||
|
||||
def _pickle_to_buffer(x):
|
||||
pickled = builtin_pickle.dumps(x, protocol=builtin_pickle.HIGHEST_PROTOCOL)
|
||||
return frombuffer(pickled)
|
||||
|
||||
|
||||
def _load_pickle_from_buffer(data):
|
||||
as_memoryview = memoryview(data)
|
||||
if six.PY2:
|
||||
return builtin_pickle.loads(as_memoryview.tobytes())
|
||||
else:
|
||||
return builtin_pickle.loads(as_memoryview)
|
||||
|
||||
|
||||
_serialize_numpy_array_pickle = _pickle_to_buffer
|
||||
_deserialize_numpy_array_pickle = _load_pickle_from_buffer
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# pandas-specific serialization matters
|
||||
|
||||
@@ -190,11 +172,3 @@ def default_serialization_context():
|
||||
|
||||
|
||||
register_default_serialization_handlers(_default_serialization_context)
|
||||
|
||||
|
||||
def pandas_serialization_context():
|
||||
context = default_serialization_context()
|
||||
context.register_type(np.ndarray, 'np.array',
|
||||
custom_serializer=_serialize_numpy_array_pickle,
|
||||
custom_deserializer=_deserialize_numpy_array_pickle)
|
||||
return context
|
||||
@@ -1613,7 +1613,7 @@ def _fully_loaded_dataframe_example():
|
||||
|
||||
|
||||
def _check_serialize_components_roundtrip(df):
|
||||
ctx = pa.pandas_serialization_context()
|
||||
ctx = pa.default_serialization_context()
|
||||
|
||||
components = ctx.serialize(df).to_components()
|
||||
deserialized = ctx.deserialize_components(components)
|
||||
|
||||
@@ -284,8 +284,6 @@ def test_clone():
|
||||
def test_primitive_serialization(large_buffer):
|
||||
for obj in PRIMITIVE_OBJECTS:
|
||||
serialization_roundtrip(obj, large_buffer)
|
||||
serialization_roundtrip(obj, large_buffer,
|
||||
pa.pandas_serialization_context())
|
||||
|
||||
|
||||
def test_serialize_to_buffer():
|
||||
|
||||
Reference in new issue
Block a user