rename to s3fs

cpp/src/arrow/filesystem/s3fs_test.cc

@@ -666,7 +666,7 @@ TEST_F(TestS3FS, OpenInputStream) {
 TEST_F(TestS3FS, OpenInputFile) {
   std::shared_ptr<io::RandomAccessFile> file;
   std::shared_ptr<Buffer> buf;
-  int64_t nbytes = -1, pos = -1;
+  int64_t nbytes = -1, pos = -1, bytes_read = 0;
 
   // Non-existent
   ASSERT_RAISES(IOError, fs_->OpenInputFile("non-existent-bucket/somefile", &file));
@@ -691,6 +691,15 @@ TEST_F(TestS3FS, OpenInputFile) {
   AssertBufferEqual(*buf, "data");
   ASSERT_OK(file->ReadAt(9, 20, &buf));
   AssertBufferEqual(*buf, "");
+
+  char result[10];
+  ASSERT_OK(file->ReadAt(2, 5, &bytes_read, &result));
+  ASSERT_EQ(bytes_read, 5);
+  ASSERT_OK(file->ReadAt(5, 20, &bytes_read, &result));
+  ASSERT_EQ(bytes_read, 4);
+  ASSERT_OK(file->ReadAt(9, 0, &bytes_read, &result));
+  ASSERT_EQ(bytes_read, 0);
+
   // Reading past end of file
   ASSERT_RAISES(IOError, file->ReadAt(10, 20, &buf));
 

python/CMakeLists.txt

@@ -386,7 +386,7 @@ set(CYTHON_EXTENSIONS lib _fs _csv _json)
 set(LINK_LIBS arrow_shared arrow_python_shared)
 
 if(PYARROW_BUILD_S3)
-  set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _s3)
+  set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _s3fs)
 endif()
 
 if(PYARROW_BUILD_CUDA)

python/pyarrow/_s3fs.pyx

@@ -23,7 +23,7 @@ from pyarrow.lib cimport check_status
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
-from pyarrow.includes.libarrow_s3 cimport *
+from pyarrow.includes.libarrow_s3fs cimport *
 from pyarrow._fs cimport FileSystem
 
 
@@ -47,16 +47,6 @@ def finalize_s3():
     check_status(CFinalizeS3())
 
 
-cdef bint _initialized = False
-
-
-cdef _ensure_initialized():
-    global _initialized
-    if not _initialized:
-        initialize_s3()
-        _initialized = True
-
-
 cdef class S3FileSystem(FileSystem):
     """S3-backed FileSystem implementation
 
@@ -92,8 +82,6 @@ cdef class S3FileSystem(FileSystem):
             CS3Options options
             shared_ptr[CS3FileSystem] wrapped
 
-        _ensure_initialized()
-
         options = CS3Options.Defaults()
         if access_key is not None or secret_key is not None:
             options.ConfigureAccessKey(tobytes(access_key),

python/pyarrow/fs.py

@@ -25,8 +25,3 @@ from pyarrow._fs import (  # noqa
     LocalFileSystem,
     SubTreeFileSystem
 )
-
-try:
-    from pyarrow._s3 import S3FileSystem, initialize_s3, finalize_s3  # noqa
-except ImportError:
-    pass

python/pyarrow/s3fs.py

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+
+from pyarrow._s3fs import (  # noqa
+    initialize_s3,
+    finalize_s3,
+    S3FileSystem
+)
+
+initialize_s3()

python/pyarrow/tests/test_fs.py

@@ -53,7 +53,7 @@ def subtree_localfs(request, tempdir, localfs):
 
 @pytest.fixture
 def s3fs(request, minio_server):
-    from pyarrow.fs import S3FileSystem
+    from pyarrow.s3fs import S3FileSystem
 
     address, access_key, secret_key = minio_server
     bucket = 'pyarrow-filesystem/'

python/setup.py

@@ -170,7 +170,6 @@ class build_ext(_build_ext):
 
     CYTHON_MODULE_NAMES = [
         'lib',
-        '_s3',
         '_fs',
         '_csv',
         '_json',
@@ -179,6 +178,7 @@ class build_ext(_build_ext):
         '_parquet',
         '_orc',
         '_plasma',
+        '_s3fs',
         'gandiva']
 
     def _run_cmake(self):
@@ -419,7 +419,7 @@ class build_ext(_build_ext):
             return True
         if name == '_flight' and not self.with_flight:
             return True
-        if name == '_s3' and not self.with_s3:
+        if name == '_s3fs' and not self.with_s3:
             return True
         if name == '_cuda' and not self.with_cuda:
             return True