MockFileSystem [skip ci]

cpp/src/arrow/python/datetime.h

@@ -113,6 +113,11 @@ inline int64_t PyDateTime_to_ns(PyDateTime_DateTime* pydatetime) {
   return PyDateTime_to_us(pydatetime) * 1000;
 }
 
+ARROW_PYTHON_EXPORT
+inline TimePoint PyDateTime_to_TimePoint(PyDateTime_DateTime* pydatetime) {
+  return TimePoint(TimePoint::duration(PyDateTime_to_ns(pydatetime)));
+}
+
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_s(PyDateTime_Delta* pytimedelta) {
   int64_t total_seconds = 0;

python/pyarrow/_fs.pxd

@@ -67,3 +67,10 @@ cdef class SubTreeFileSystem(FileSystem):
         CSubTreeFileSystem* subtreefs
 
     cdef init(self, const shared_ptr[CFileSystem]& wrapped)
+
+
+cdef class _MockFileSystem(FileSystem):
+    cdef:
+        CMockFileSystem* mockfs
+
+    cdef init(self, const shared_ptr[CFileSystem]& wrapped)

python/pyarrow/_fs.pyx

@@ -18,10 +18,13 @@
 # cython: language_level = 3
 
 import six
+from cpython.datetime cimport datetime, PyDateTime_DateTime
 
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.includes.common cimport *
-from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint
+from pyarrow.includes.libarrow cimport (
+    PyDateTime_from_TimePoint, PyDateTime_to_TimePoint
+)
 from pyarrow.lib import _detect_compression
 from pyarrow.lib cimport *
 
@@ -547,3 +550,20 @@ cdef class SubTreeFileSystem(FileSystem):
     cdef init(self, const shared_ptr[CFileSystem]& wrapped):
         FileSystem.init(self, wrapped)
         self.subtreefs = <CSubTreeFileSystem*> wrapped.get()
+
+
+cdef class _MockFileSystem(FileSystem):
+
+    def __init__(self, datetime current_time=None):
+        cdef shared_ptr[CMockFileSystem] wrapped
+
+        current_time = current_time or datetime.now()
+        wrapped = make_shared[CMockFileSystem](
+            PyDateTime_to_TimePoint(<PyDateTime_DateTime*> current_time)
+        )
+
+        self.init(<shared_ptr[CFileSystem]> wrapped)
+
+    cdef init(self, const shared_ptr[CFileSystem]& wrapped):
+        FileSystem.init(self, wrapped)
+        self.mockfs = <CMockFileSystem*> wrapped.get()

python/pyarrow/includes/common.pxd

@@ -27,6 +27,7 @@ from libcpp.unordered_map cimport unordered_map
 from libcpp.unordered_set cimport unordered_set
 
 from cpython cimport PyObject
+from cpython.datetime cimport PyDateTime_DateTime
 cimport cpython
 
 

python/pyarrow/includes/libarrow.pxd

@@ -1572,13 +1572,13 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
 
 
 cdef extern from "arrow/python/api.h" namespace "arrow::py::internal" nogil:
-
     cdef cppclass CTimePoint "arrow::py::internal::TimePoint":
         pass
 
-    cdef CStatus PyDateTime_from_int(int64_t val, const TimeUnit unit,
-                                     PyObject** out)
-    cdef CStatus PyDateTime_from_TimePoint(CTimePoint val, PyObject** out)
+    CStatus PyDateTime_from_int(int64_t val, const TimeUnit unit,
+                                PyObject** out)
+    CStatus PyDateTime_from_TimePoint(CTimePoint val, PyObject** out)
+    CTimePoint PyDateTime_to_TimePoint(PyDateTime_DateTime* pydatetime)
 
 
 cdef extern from 'arrow/python/init.h':

python/pyarrow/includes/libarrow_dataset.pxd

@@ -143,12 +143,9 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
         c_string file_type()
 
     cdef cppclass CFileSource "arrow::dataset::FileSource":
-        # enum SourceType { PATH, BUFFER };
         CFileSource(c_string path, CFileSystem filesystem,
                     CompressionType compression)
-        CFileSource(shared_ptr[CBuffer] buffer, CompressionType compression)
         c_bool operator==(const CFileSource other)
-        # CSourceType type()
         CompressionType compression()
         c_string path()
         CFileSystem* filesystem()
@@ -161,11 +158,13 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
 
     cdef cppclass CFileFormat "arrow::dataset::FileFormat":
         c_string name()
-        c_bool IsKnownExtension(const c_string& ext)
+        CStatus IsSupported(const CFileSource& source, c_bool* supported) const
+        CStatus Inspect(const CFileSource& source,
+                        shared_ptr[CSchema]* out) const
         CStatus ScanFile(const CFileSource& source,
                          shared_ptr[CScanOptions] scan_options,
                          shared_ptr[CScanContext] scan_context,
-                         unique_ptr[CScanTaskIterator]* out)
+                         CScanTaskIterator* out) const
         CStatus MakeFragment(const CFileSource& location,
                              shared_ptr[CScanOptions] opts,
                              unique_ptr[CDataFragment]* out)

python/pyarrow/includes/libarrow_fs.pxd

@@ -147,3 +147,7 @@ cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil:
         @staticmethod
         CResult[shared_ptr[CHadoopFileSystem]] Make(
             const CHdfsOptions& options)
+
+    cdef cppclass CMockFileSystem "arrow::fs::internal::MockFileSystem"(
+            CFileSystem):
+        CMockFileSystem(CTimePoint current_time)