mirror of
https://github.com/apache/arrow.git
synced 2026-10-29 09:05:47 +08:00
resolve comments
This commit is contained in:
1 parent
fc801756d2
commit
de8529cf18
5 files changed
+49
-45
No files matched your search
@@ -136,7 +136,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_arrow_adapter_orc_OrcReaderJniWrapper_op
|
||||
|
||||
arrow::Status ret;
|
||||
if (path.find("hdfs://") == 0) {
|
||||
env->ThrowNew(io_exception_class, "hdfs path not support yet.");
|
||||
env->ThrowNew(io_exception_class, "hdfs path not supported yet.");
|
||||
} else {
|
||||
ret = arrow::io::ReadableFile::Open(path, &in_file);
|
||||
}
|
||||
@@ -183,6 +183,11 @@ Java_org_apache_arrow_adapter_orc_OrcReaderJniWrapper_nextStripeReader(JNIEnv* e
|
||||
jlong id,
|
||||
jlong batch_size) {
|
||||
auto reader = orc_reader_holder_.Lookup(id);
|
||||
if (!reader) {
|
||||
std::string error_message = "invalid reader id " + std::to_string(id);
|
||||
env->ThrowNew(exception_class, error_message.c_str());
|
||||
}
|
||||
|
||||
std::shared_ptr<RecordBatchReader> stripe_reader;
|
||||
auto status = reader->NextStripeReader(batch_size, &stripe_reader);
|
||||
|
||||
@@ -202,6 +207,11 @@ Java_org_apache_arrow_adapter_orc_OrcStripeReaderJniWrapper_getSchema(JNIEnv* en
|
||||
jclass this_cls,
|
||||
jlong id) {
|
||||
auto stripe_reader = orc_stripe_reader_holder_.Lookup(id);
|
||||
if (!stripe_reader) {
|
||||
std::string error_message = "invalid stripe reader id " + std::to_string(id);
|
||||
env->ThrowNew(exception_class, error_message.c_str());
|
||||
}
|
||||
|
||||
auto schema = stripe_reader->schema();
|
||||
|
||||
std::shared_ptr<arrow::Buffer> out;
|
||||
@@ -222,6 +232,11 @@ Java_org_apache_arrow_adapter_orc_OrcStripeReaderJniWrapper_next(JNIEnv* env,
|
||||
jclass this_cls,
|
||||
jlong id) {
|
||||
auto stripe_reader = orc_stripe_reader_holder_.Lookup(id);
|
||||
if (!stripe_reader) {
|
||||
std::string error_message = "invalid stripe reader id " + std::to_string(id);
|
||||
env->ThrowNew(exception_class, error_message.c_str());
|
||||
}
|
||||
|
||||
std::shared_ptr<arrow::RecordBatch> record_batch;
|
||||
auto status = stripe_reader->ReadNext(&record_batch);
|
||||
if (!status.ok() || !record_batch) {
|
||||
|
||||
+5
-5
@@ -22,7 +22,7 @@ package org.apache.arrow.adapter.orc;
|
||||
*/
|
||||
class OrcMemoryJniWrapper implements AutoCloseable {
|
||||
|
||||
private final long id;
|
||||
private final long nativeInstanceId;
|
||||
|
||||
private final long memoryAddress;
|
||||
|
||||
@@ -32,13 +32,13 @@ class OrcMemoryJniWrapper implements AutoCloseable {
|
||||
|
||||
/**
|
||||
* Construct a new instance.
|
||||
* @param id unique id of the underlying memory.
|
||||
* @param nativeInstanceId unique id of the underlying memory.
|
||||
* @param memoryAddress starting memory address of the the underlying memory.
|
||||
* @param size size of the valid data.
|
||||
* @param capacity allocated memory size.
|
||||
*/
|
||||
OrcMemoryJniWrapper(long id, long memoryAddress, long size, long capacity) {
|
||||
this.id = id;
|
||||
OrcMemoryJniWrapper(long nativeInstanceId, long memoryAddress, long size, long capacity) {
|
||||
this.nativeInstanceId = nativeInstanceId;
|
||||
this.memoryAddress = memoryAddress;
|
||||
this.size = size;
|
||||
this.capacity = capacity;
|
||||
@@ -70,7 +70,7 @@ class OrcMemoryJniWrapper implements AutoCloseable {
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
release(id);
|
||||
release(nativeInstanceId);
|
||||
}
|
||||
|
||||
private native void release(long id);
|
||||
|
||||
@@ -35,7 +35,7 @@ public class OrcReader implements AutoCloseable {
|
||||
/**
|
||||
* reference to native reader instance.
|
||||
*/
|
||||
private final long id;
|
||||
private final long nativeInstanceId;
|
||||
|
||||
/**
|
||||
* Create an OrcReader that iterate over orc stripes.
|
||||
@@ -46,7 +46,7 @@ public class OrcReader implements AutoCloseable {
|
||||
public OrcReader(String filePath, BufferAllocator allocator) throws IOException {
|
||||
this.allocator = allocator;
|
||||
this.jniWrapper = OrcReaderJniWrapper.getInstance();
|
||||
this.id = jniWrapper.open(filePath);
|
||||
this.nativeInstanceId = jniWrapper.open(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -56,7 +56,7 @@ public class OrcReader implements AutoCloseable {
|
||||
* @return true if seek operation is succeeded
|
||||
*/
|
||||
public boolean seek(int rowNumber) {
|
||||
return jniWrapper.seek(id, rowNumber);
|
||||
return jniWrapper.seek(nativeInstanceId, rowNumber);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -66,7 +66,7 @@ public class OrcReader implements AutoCloseable {
|
||||
* @return ArrowReader that iterate over current stripes
|
||||
*/
|
||||
public ArrowReader nextStripeReader(long batchSize) {
|
||||
long stripeReaderId = jniWrapper.nextStripeReader(id, batchSize);
|
||||
long stripeReaderId = jniWrapper.nextStripeReader(nativeInstanceId, batchSize);
|
||||
if (stripeReaderId < 0) {
|
||||
return null;
|
||||
}
|
||||
@@ -80,11 +80,11 @@ public class OrcReader implements AutoCloseable {
|
||||
* @return number of stripes
|
||||
*/
|
||||
public int getNumberOfStripes() {
|
||||
return jniWrapper.getNumberOfStripes(id);
|
||||
return jniWrapper.getNumberOfStripes(nativeInstanceId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
jniWrapper.close(id);
|
||||
jniWrapper.close(nativeInstanceId);
|
||||
}
|
||||
}
|
||||
+1
-12
@@ -102,18 +102,7 @@ public class OrcReferenceManager implements ReferenceManager {
|
||||
|
||||
@Override
|
||||
public OwnershipTransferResult transferOwnership(ArrowBuf sourceBuffer, BufferAllocator targetAllocator) {
|
||||
retain();
|
||||
return new OwnershipTransferResult() {
|
||||
@Override
|
||||
public boolean getAllocationFit() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ArrowBuf getTransferredBuffer() {
|
||||
return sourceBuffer;
|
||||
}
|
||||
};
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -43,23 +43,22 @@ public class OrcStripeReader extends ArrowReader {
|
||||
/**
|
||||
* reference to native stripe reader instance.
|
||||
*/
|
||||
private final long id;
|
||||
private MessageChannelReader schemaReader;
|
||||
private final long nativeInstanceId;
|
||||
|
||||
/**
|
||||
* Construct a new instance.
|
||||
* @param id id of the stripe reader instance, obtained by
|
||||
* @param nativeInstanceId nativeInstanceId of the stripe reader instance, obtained by
|
||||
* calling nextStripeReader from OrcReaderJniWrapper
|
||||
* @param allocator memory allocator for accounting.
|
||||
*/
|
||||
OrcStripeReader(long id, BufferAllocator allocator) {
|
||||
OrcStripeReader(long nativeInstanceId, BufferAllocator allocator) {
|
||||
super(allocator);
|
||||
this.id = id;
|
||||
this.nativeInstanceId = nativeInstanceId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean loadNextBatch() throws IOException {
|
||||
OrcRecordBatch recordBatch = OrcStripeReaderJniWrapper.next(id);
|
||||
OrcRecordBatch recordBatch = OrcStripeReaderJniWrapper.next(nativeInstanceId);
|
||||
if (recordBatch == null) {
|
||||
return false;
|
||||
}
|
||||
@@ -91,28 +90,29 @@ public class OrcStripeReader extends ArrowReader {
|
||||
|
||||
@Override
|
||||
protected void closeReadSource() throws IOException {
|
||||
OrcStripeReaderJniWrapper.close(id);
|
||||
OrcStripeReaderJniWrapper.close(nativeInstanceId);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Schema readSchema() throws IOException {
|
||||
byte[] schemaBytes = OrcStripeReaderJniWrapper.getSchema(id);
|
||||
schemaReader = new MessageChannelReader(
|
||||
new ReadChannel(
|
||||
new ByteArrayReadableSeekableByteChannel(schemaBytes)), allocator);
|
||||
byte[] schemaBytes = OrcStripeReaderJniWrapper.getSchema(nativeInstanceId);
|
||||
|
||||
MessageResult result = schemaReader.readNext();
|
||||
schemaReader.close();
|
||||
try (MessageChannelReader schemaReader =
|
||||
new MessageChannelReader(
|
||||
new ReadChannel(
|
||||
new ByteArrayReadableSeekableByteChannel(schemaBytes)), allocator)) {
|
||||
|
||||
if (result == null) {
|
||||
throw new IOException("Unexpected end of input. Missing schema.");
|
||||
MessageResult result = schemaReader.readNext();
|
||||
if (result == null) {
|
||||
throw new IOException("Unexpected end of input. Missing schema.");
|
||||
}
|
||||
|
||||
if (result.getMessage().headerType() != MessageHeader.Schema) {
|
||||
throw new IOException("Expected schema but header was " + result.getMessage().headerType());
|
||||
}
|
||||
|
||||
return MessageSerializer.deserializeSchema(result.getMessage());
|
||||
}
|
||||
|
||||
if (result.getMessage().headerType() != MessageHeader.Schema) {
|
||||
throw new IOException("Expected schema but header was " + result.getMessage().headerType());
|
||||
}
|
||||
|
||||
return MessageSerializer.deserializeSchema(result.getMessage());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
Reference in new issue
Block a user