resolve comments

This commit is contained in:
Yurui Zhou committed 2019-06-06 15:12:01 +08:00
1 parent fc801756d2
commit de8529cf18
5 files changed
+49 -45

No files matched your search

+16 -1
View File
@@ -136,7 +136,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_arrow_adapter_orc_OrcReaderJniWrapper_op
arrow::Status ret;
if (path.find("hdfs://") == 0) {
env->ThrowNew(io_exception_class, "hdfs path not support yet.");
env->ThrowNew(io_exception_class, "hdfs path not supported yet.");
} else {
ret = arrow::io::ReadableFile::Open(path, &in_file);
}
@@ -183,6 +183,11 @@ Java_org_apache_arrow_adapter_orc_OrcReaderJniWrapper_nextStripeReader(JNIEnv* e
jlong id,
jlong batch_size) {
auto reader = orc_reader_holder_.Lookup(id);
if (!reader) {
std::string error_message = "invalid reader id " + std::to_string(id);
env->ThrowNew(exception_class, error_message.c_str());
}
std::shared_ptr<RecordBatchReader> stripe_reader;
auto status = reader->NextStripeReader(batch_size, &stripe_reader);
@@ -202,6 +207,11 @@ Java_org_apache_arrow_adapter_orc_OrcStripeReaderJniWrapper_getSchema(JNIEnv* en
jclass this_cls,
jlong id) {
auto stripe_reader = orc_stripe_reader_holder_.Lookup(id);
if (!stripe_reader) {
std::string error_message = "invalid stripe reader id " + std::to_string(id);
env->ThrowNew(exception_class, error_message.c_str());
}
auto schema = stripe_reader->schema();
std::shared_ptr<arrow::Buffer> out;
@@ -222,6 +232,11 @@ Java_org_apache_arrow_adapter_orc_OrcStripeReaderJniWrapper_next(JNIEnv* env,
jclass this_cls,
jlong id) {
auto stripe_reader = orc_stripe_reader_holder_.Lookup(id);
if (!stripe_reader) {
std::string error_message = "invalid stripe reader id " + std::to_string(id);
env->ThrowNew(exception_class, error_message.c_str());
}
std::shared_ptr<arrow::RecordBatch> record_batch;
auto status = stripe_reader->ReadNext(&record_batch);
if (!status.ok() || !record_batch) {
@@ -22,7 +22,7 @@ package org.apache.arrow.adapter.orc;
*/
class OrcMemoryJniWrapper implements AutoCloseable {
private final long id;
private final long nativeInstanceId;
private final long memoryAddress;
@@ -32,13 +32,13 @@ class OrcMemoryJniWrapper implements AutoCloseable {
/**
* Construct a new instance.
* @param id unique id of the underlying memory.
* @param nativeInstanceId unique id of the underlying memory.
* @param memoryAddress starting memory address of the the underlying memory.
* @param size size of the valid data.
* @param capacity allocated memory size.
*/
OrcMemoryJniWrapper(long id, long memoryAddress, long size, long capacity) {
this.id = id;
OrcMemoryJniWrapper(long nativeInstanceId, long memoryAddress, long size, long capacity) {
this.nativeInstanceId = nativeInstanceId;
this.memoryAddress = memoryAddress;
this.size = size;
this.capacity = capacity;
@@ -70,7 +70,7 @@ class OrcMemoryJniWrapper implements AutoCloseable {
@Override
public void close() {
release(id);
release(nativeInstanceId);
}
private native void release(long id);
@@ -35,7 +35,7 @@ public class OrcReader implements AutoCloseable {
/**
* reference to native reader instance.
*/
private final long id;
private final long nativeInstanceId;
/**
* Create an OrcReader that iterate over orc stripes.
@@ -46,7 +46,7 @@ public class OrcReader implements AutoCloseable {
public OrcReader(String filePath, BufferAllocator allocator) throws IOException {
this.allocator = allocator;
this.jniWrapper = OrcReaderJniWrapper.getInstance();
this.id = jniWrapper.open(filePath);
this.nativeInstanceId = jniWrapper.open(filePath);
}
/**
@@ -56,7 +56,7 @@ public class OrcReader implements AutoCloseable {
* @return true if seek operation is succeeded
*/
public boolean seek(int rowNumber) {
return jniWrapper.seek(id, rowNumber);
return jniWrapper.seek(nativeInstanceId, rowNumber);
}
/**
@@ -66,7 +66,7 @@ public class OrcReader implements AutoCloseable {
* @return ArrowReader that iterate over current stripes
*/
public ArrowReader nextStripeReader(long batchSize) {
long stripeReaderId = jniWrapper.nextStripeReader(id, batchSize);
long stripeReaderId = jniWrapper.nextStripeReader(nativeInstanceId, batchSize);
if (stripeReaderId < 0) {
return null;
}
@@ -80,11 +80,11 @@ public class OrcReader implements AutoCloseable {
* @return number of stripes
*/
public int getNumberOfStripes() {
return jniWrapper.getNumberOfStripes(id);
return jniWrapper.getNumberOfStripes(nativeInstanceId);
}
@Override
public void close() throws Exception {
jniWrapper.close(id);
jniWrapper.close(nativeInstanceId);
}
}
@@ -102,18 +102,7 @@ public class OrcReferenceManager implements ReferenceManager {
@Override
public OwnershipTransferResult transferOwnership(ArrowBuf sourceBuffer, BufferAllocator targetAllocator) {
retain();
return new OwnershipTransferResult() {
@Override
public boolean getAllocationFit() {
return false;
}
@Override
public ArrowBuf getTransferredBuffer() {
return sourceBuffer;
}
};
throw new UnsupportedOperationException();
}
@Override
@@ -43,23 +43,22 @@ public class OrcStripeReader extends ArrowReader {
/**
* reference to native stripe reader instance.
*/
private final long id;
private MessageChannelReader schemaReader;
private final long nativeInstanceId;
/**
* Construct a new instance.
* @param id id of the stripe reader instance, obtained by
* @param nativeInstanceId nativeInstanceId of the stripe reader instance, obtained by
* calling nextStripeReader from OrcReaderJniWrapper
* @param allocator memory allocator for accounting.
*/
OrcStripeReader(long id, BufferAllocator allocator) {
OrcStripeReader(long nativeInstanceId, BufferAllocator allocator) {
super(allocator);
this.id = id;
this.nativeInstanceId = nativeInstanceId;
}
@Override
public boolean loadNextBatch() throws IOException {
OrcRecordBatch recordBatch = OrcStripeReaderJniWrapper.next(id);
OrcRecordBatch recordBatch = OrcStripeReaderJniWrapper.next(nativeInstanceId);
if (recordBatch == null) {
return false;
}
@@ -91,28 +90,29 @@ public class OrcStripeReader extends ArrowReader {
@Override
protected void closeReadSource() throws IOException {
OrcStripeReaderJniWrapper.close(id);
OrcStripeReaderJniWrapper.close(nativeInstanceId);
}
@Override
protected Schema readSchema() throws IOException {
byte[] schemaBytes = OrcStripeReaderJniWrapper.getSchema(id);
schemaReader = new MessageChannelReader(
new ReadChannel(
new ByteArrayReadableSeekableByteChannel(schemaBytes)), allocator);
byte[] schemaBytes = OrcStripeReaderJniWrapper.getSchema(nativeInstanceId);
MessageResult result = schemaReader.readNext();
schemaReader.close();
try (MessageChannelReader schemaReader =
new MessageChannelReader(
new ReadChannel(
new ByteArrayReadableSeekableByteChannel(schemaBytes)), allocator)) {
if (result == null) {
throw new IOException("Unexpected end of input. Missing schema.");
MessageResult result = schemaReader.readNext();
if (result == null) {
throw new IOException("Unexpected end of input. Missing schema.");
}
if (result.getMessage().headerType() != MessageHeader.Schema) {
throw new IOException("Expected schema but header was " + result.getMessage().headerType());
}
return MessageSerializer.deserializeSchema(result.getMessage());
}
if (result.getMessage().headerType() != MessageHeader.Schema) {
throw new IOException("Expected schema but header was " + result.getMessage().headerType());
}
return MessageSerializer.deserializeSchema(result.getMessage());
}
@Override