mirror of
https://github.com/apache/arrow.git
synced 2026-10-28 16:47:19 +08:00
get_buffer -> get_buffers and update example
This commit is contained in:
1 parent
8c36903755
commit
e60ea73e09
3 files changed
+12
-12
No files matched your search
@@ -81,7 +81,7 @@ def put_df(df):
|
||||
|
||||
def get_dfs(object_ids):
|
||||
"""Retrieve dataframes from the object store given their object IDs."""
|
||||
buffers = client.get(object_ids)
|
||||
buffers = client.get_buffers(object_ids)
|
||||
return [pa.RecordBatchStreamReader(buf).read_next_batch().to_pandas()
|
||||
for buf in buffers]
|
||||
|
||||
|
||||
@@ -303,7 +303,7 @@ cdef class PlasmaClient:
|
||||
metadata.size(), &data))
|
||||
return self._make_mutable_plasma_buffer(object_id, data, data_size)
|
||||
|
||||
def get_buffer(self, object_ids, timeout_ms=-1):
|
||||
def get_buffers(self, object_ids, timeout_ms=-1):
|
||||
"""
|
||||
Returns data buffer from the PlasmaStore based on object ID.
|
||||
|
||||
@@ -419,7 +419,7 @@ cdef class PlasmaClient:
|
||||
"""
|
||||
if isinstance(object_ids, collections.Sequence):
|
||||
results = []
|
||||
buffers = self.get_buffer(object_ids, timeout_ms)
|
||||
buffers = self.get_buffers(object_ids, timeout_ms)
|
||||
for i in range(len(object_ids)):
|
||||
# buffers[i] is None if this object was not available within the
|
||||
# timeout
|
||||
|
||||
@@ -82,8 +82,8 @@ def create_object(client, data_size, metadata_size, seal=True):
|
||||
def assert_get_object_equal(unit_test, client1, client2, object_id,
|
||||
memory_buffer=None, metadata=None):
|
||||
import pyarrow.plasma as plasma
|
||||
client1_buff = client1.get_buffer([object_id])[0]
|
||||
client2_buff = client2.get_buffer([object_id])[0]
|
||||
client1_buff = client1.get_buffers([object_id])[0]
|
||||
client2_buff = client2.get_buffers([object_id])[0]
|
||||
client1_metadata = client1.get_metadata([object_id])[0]
|
||||
client2_metadata = client2.get_metadata([object_id])[0]
|
||||
assert len(client1_buff) == len(client2_buff)
|
||||
@@ -187,7 +187,7 @@ class TestPlasmaClient(object):
|
||||
# Seal the object.
|
||||
self.plasma_client.seal(object_id)
|
||||
# Get the object.
|
||||
memory_buffer = np.frombuffer(self.plasma_client.get_buffer([object_id])[0],
|
||||
memory_buffer = np.frombuffer(self.plasma_client.get_buffers([object_id])[0],
|
||||
dtype="uint8")
|
||||
for i in range(length):
|
||||
assert memory_buffer[i] == i % 256
|
||||
@@ -209,7 +209,7 @@ class TestPlasmaClient(object):
|
||||
self.plasma_client.seal(object_id)
|
||||
# Get the object.
|
||||
memory_buffer = np.frombuffer(
|
||||
self.plasma_client.get_buffer([object_id])[0], dtype="uint8")
|
||||
self.plasma_client.get_buffers([object_id])[0], dtype="uint8")
|
||||
for i in range(length):
|
||||
assert memory_buffer[i] == i % 256
|
||||
# Get the metadata.
|
||||
@@ -241,7 +241,7 @@ class TestPlasmaClient(object):
|
||||
# Test timing out of get with various timeouts.
|
||||
for timeout in [0, 10, 100, 1000]:
|
||||
object_ids = [random_object_id() for _ in range(num_object_ids)]
|
||||
results = self.plasma_client.get_buffer(object_ids, timeout_ms=timeout)
|
||||
results = self.plasma_client.get_buffers(object_ids, timeout_ms=timeout)
|
||||
assert results == num_object_ids * [None]
|
||||
|
||||
data_buffers = []
|
||||
@@ -256,7 +256,7 @@ class TestPlasmaClient(object):
|
||||
# Test timing out from some but not all get calls with various
|
||||
# timeouts.
|
||||
for timeout in [0, 10, 100, 1000]:
|
||||
data_results = self.plasma_client.get_buffer(object_ids,
|
||||
data_results = self.plasma_client.get_buffers(object_ids,
|
||||
timeout_ms=timeout)
|
||||
# metadata_results = self.plasma_client.get_metadata(
|
||||
# object_ids, timeout_ms=timeout)
|
||||
@@ -297,7 +297,7 @@ class TestPlasmaClient(object):
|
||||
pa.write_tensor(tensor, stream)
|
||||
self.plasma_client.seal(object_id)
|
||||
# Read the arrow object.
|
||||
[tensor] = self.plasma_client.get_buffer([object_id])
|
||||
[tensor] = self.plasma_client.get_buffers([object_id])
|
||||
reader = pa.BufferReader(tensor)
|
||||
array = pa.read_tensor(reader).to_numpy()
|
||||
# Assert that they are equal.
|
||||
@@ -326,7 +326,7 @@ class TestPlasmaClient(object):
|
||||
self.plasma_client.seal(object_id)
|
||||
|
||||
# Read the DataFrame.
|
||||
[data] = self.plasma_client.get_buffer([object_id])
|
||||
[data] = self.plasma_client.get_buffers([object_id])
|
||||
reader = pa.RecordBatchStreamReader(pa.BufferReader(data))
|
||||
result = reader.get_next_batch().to_pandas()
|
||||
|
||||
@@ -564,7 +564,7 @@ class TestPlasmaClient(object):
|
||||
# with pytest.raises(Exception):
|
||||
# illegal_assignment()
|
||||
# Get the object.
|
||||
memory_buffer = self.plasma_client.get_buffer([object_id])[0]
|
||||
memory_buffer = self.plasma_client.get_buffers([object_id])[0]
|
||||
|
||||
# Make sure the object is read only.
|
||||
def illegal_assignment():
|
||||
|
||||
Reference in new issue
Block a user