type_name

cpp/src/arrow/dataset/dataset.h

@@ -44,8 +44,6 @@ class ARROW_DS_EXPORT DataFragment {
   /// scanning
   virtual bool splittable() const = 0;
 
-  virtual std::string type() const = 0;
-
   /// \brief Filtering, schema reconciliation, and partition options to use when
   /// scanning this fragment. May be nullptr, which indicates that no filtering
   /// or schema reconciliation will be performed and all partitions will be
@@ -78,8 +76,6 @@ class ARROW_DS_EXPORT SimpleDataFragment : public DataFragment {
   SimpleDataFragment(std::vector<std::shared_ptr<RecordBatch>> record_batches,
                      ScanOptionsPtr scan_options);
 
-  std::string type() const override { return "simple_data_fragment"; }
-
   Result<ScanTaskIterator> Scan(ScanContextPtr context) override;
 
   bool splittable() const override { return false; }
@@ -101,7 +97,7 @@ class ARROW_DS_EXPORT DataSource {
   /// May be null, which indicates no information is available.
   const ExpressionPtr& partition_expression() const { return partition_expression_; }
 
-  virtual std::string type() const = 0;
+  virtual std::string type_name() const = 0;
 
   virtual ~DataSource() = default;
 
@@ -135,7 +131,7 @@ class ARROW_DS_EXPORT SimpleDataSource : public DataSource {
 
   DataFragmentIterator GetFragmentsImpl(ScanOptionsPtr options) override;
 
-  std::string type() const override { return "simple_data_source"; }
+  std::string type_name() const override { return "simple"; }
 
  private:
   DataFragmentVector fragments_;
@@ -148,7 +144,7 @@ class ARROW_DS_EXPORT TreeDataSource : public DataSource {
 
   DataFragmentIterator GetFragmentsImpl(ScanOptionsPtr options) override;
 
-  std::string type() const override { return "tree_data_source"; }
+  std::string type_name() const override { return "tree"; }
 
  private:
   DataSourceVector children_;

cpp/src/arrow/dataset/file_base.h

@@ -216,7 +216,7 @@ class ARROW_DS_EXPORT FileSystemDataSource : public DataSource {
                                     ExpressionVector partitions,
                                     ExpressionPtr source_partition, FileFormatPtr format);
 
-  std::string type() const override { return "filesystem_data_source"; }
+  std::string type_name() const override { return "filesystem"; }
 
   std::string ToString() const;
 

cpp/src/arrow/dataset/file_parquet.h

@@ -70,8 +70,6 @@ class ARROW_DS_EXPORT ParquetFragment : public FileDataFragment {
   ParquetFragment(const FileSource& source, ScanOptionsPtr options)
       : FileDataFragment(source, std::make_shared<ParquetFileFormat>(), options) {}
 
-  std::string type() const override { return "parquet_data_fragment"; }
-
   bool splittable() const override { return true; }
 };
 

cpp/src/arrow/dataset/partition.h

@@ -60,7 +60,7 @@ class ARROW_DS_EXPORT PartitionScheme {
   virtual ~PartitionScheme() = default;
 
   /// \brief The name identifying the kind of partition scheme
-  virtual std::string name() const = 0;
+  virtual std::string type_name() const = 0;
 
   /// \brief Parse a path segment into a partition expression
   ///
@@ -113,7 +113,7 @@ class DefaultPartitionScheme : public PartitionScheme {
  public:
   DefaultPartitionScheme() : PartitionScheme(::arrow::schema({})) {}
 
-  std::string name() const override { return "default_partition_scheme"; }
+  std::string type_name() const override { return "default"; }
 
   Result<ExpressionPtr> Parse(const std::string& segment, int i) const override {
     return scalar(true);
@@ -129,7 +129,7 @@ class ARROW_DS_EXPORT SegmentDictionaryPartitionScheme : public PartitionScheme
       std::vector<std::unordered_map<std::string, ExpressionPtr>> dictionaries)
       : PartitionScheme(std::move(schema)), dictionaries_(std::move(dictionaries)) {}
 
-  std::string name() const override { return "segment_dictionary_partition_scheme"; }
+  std::string type_name() const override { return "segment_dictionary"; }
 
   /// Return dictionaries_[i][segment] or scalar(true)
   Result<ExpressionPtr> Parse(const std::string& segment, int i) const override;
@@ -172,7 +172,7 @@ class ARROW_DS_EXPORT SchemaPartitionScheme : public PartitionKeysScheme {
   explicit SchemaPartitionScheme(std::shared_ptr<Schema> schema)
       : PartitionKeysScheme(std::move(schema)) {}
 
-  std::string name() const override { return "schema_partition_scheme"; }
+  std::string type_name() const override { return "schema"; }
 
   util::optional<Key> ParseKey(const std::string& segment, int i) const override;
 
@@ -193,7 +193,7 @@ class ARROW_DS_EXPORT HivePartitionScheme : public PartitionKeysScheme {
   explicit HivePartitionScheme(std::shared_ptr<Schema> schema)
       : PartitionKeysScheme(std::move(schema)) {}
 
-  std::string name() const override { return "hive_partition_scheme"; }
+  std::string type_name() const override { return "hive"; }
 
   util::optional<Key> ParseKey(const std::string& segment, int i) const override {
     return ParseKey(segment);
@@ -210,12 +210,12 @@ class ARROW_DS_EXPORT FunctionPartitionScheme : public PartitionScheme {
   explicit FunctionPartitionScheme(
       std::shared_ptr<Schema> schema,
       std::function<Result<ExpressionPtr>(const std::string&, int)> impl,
-      std::string name = "function_partition_scheme")
+      std::string name = "function")
       : PartitionScheme(std::move(schema)),
         impl_(std::move(impl)),
         name_(std::move(name)) {}
 
-  std::string name() const override { return name_; }
+  std::string type_name() const override { return name_; }
 
   Result<ExpressionPtr> Parse(const std::string& segment, int i) const override {
     return impl_(segment, i);

cpp/src/arrow/dataset/partition_test.cc

@@ -229,7 +229,7 @@ class RangePartitionScheme : public HivePartitionScheme {
  public:
   using HivePartitionScheme::HivePartitionScheme;
 
-  std::string name() const override { return "range_partition_scheme"; }
+  std::string type_name() const override { return "range"; }
 
   Result<ExpressionPtr> Parse(const std::string& segment, int i) const override {
     ExpressionVector ranges;

cpp/src/arrow/dataset/test_util.h

@@ -205,8 +205,6 @@ class DummyFragment : public FileDataFragment {
   DummyFragment(const FileSource& source, ScanOptionsPtr options)
       : FileDataFragment(source, std::make_shared<DummyFileFormat>(), options) {}
 
-  std::string type() const override { return "dummy_data_fragment"; }
-
   bool splittable() const override { return false; }
 };
 
@@ -263,8 +261,6 @@ class JSONRecordBatchFragment : public FileDataFragment {
       : FileDataFragment(source, std::make_shared<JSONRecordBatchFileFormat>(schema),
                          options) {}
 
-  std::string type() const override { return "json_batch_data_fragment"; }
-
   bool splittable() const override { return false; }
 };
 

cpp/src/arrow/filesystem/filesystem.h

@@ -155,7 +155,7 @@ class ARROW_EXPORT FileSystem {
  public:
   virtual ~FileSystem();
 
-  virtual std::string type() const = 0;
+  virtual std::string type_name() const = 0;
 
   /// Get statistics for the given target.
   ///
@@ -245,7 +245,7 @@ class ARROW_EXPORT SubTreeFileSystem : public FileSystem {
                              std::shared_ptr<FileSystem> base_fs);
   ~SubTreeFileSystem() override;
 
-  std::string type() const override { return "subtree"; }
+  std::string type_name() const override { return "subtree"; }
 
   /// \cond FALSE
   using FileSystem::GetTargetStats;
@@ -293,7 +293,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {
   SlowFileSystem(std::shared_ptr<FileSystem> base_fs, double average_latency,
                  int32_t seed);
 
-  std::string type() const override { return "slow"; }
+  std::string type_name() const override { return "slow"; }
 
   using FileSystem::GetTargetStats;
   Result<FileStats> GetTargetStats(const std::string& path) override;

cpp/src/arrow/filesystem/hdfs.h

@@ -61,7 +61,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
  public:
   ~HadoopFileSystem() override;
 
-  std::string type() const override { return "hdfs"; }
+  std::string type_name() const override { return "hdfs"; }
 
   /// \cond FALSE
   using FileSystem::GetTargetStats;

cpp/src/arrow/filesystem/localfs.h

@@ -48,7 +48,7 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
   explicit LocalFileSystem(const LocalFileSystemOptions&);
   ~LocalFileSystem() override;
 
-  std::string type() const override { return "local"; }
+  std::string type_name() const override { return "local"; }
 
   /// \cond FALSE
   using FileSystem::GetTargetStats;

cpp/src/arrow/filesystem/mockfs.h

@@ -60,7 +60,7 @@ class ARROW_EXPORT MockFileSystem : public FileSystem {
   explicit MockFileSystem(TimePoint current_time);
   ~MockFileSystem() override;
 
-  std::string type() const override { return "mock"; }
+  std::string type_name() const override { return "mock"; }
 
   // XXX It's not very practical to have to explicitly declare inheritance
   // of default overrides.

cpp/src/arrow/filesystem/s3fs.h

@@ -79,7 +79,7 @@ class ARROW_EXPORT S3FileSystem : public FileSystem {
  public:
   ~S3FileSystem() override;
 
-  std::string type() const override { return "s3"; }
+  std::string type_name() const override { return "s3"; }
 
   /// \cond FALSE
   using FileSystem::GetTargetStats;

python/pyarrow/_dataset.pyx

@@ -95,13 +95,12 @@ cdef class PartitionScheme:
     cdef wrap(const shared_ptr[CPartitionScheme]& sp):
         cdef PartitionScheme self
 
-        # TODO(kszucs): either choose type() or name() but consistently
-        typ = frombytes(sp.get().name())
-        if typ == 'default_partition_scheme':
+        typ = frombytes(sp.get().type_name())
+        if typ == 'default':
             self = DefaultPartitionScheme.__new__(DefaultPartitionScheme)
-        elif typ == 'schema_partition_scheme':
+        elif typ == 'schema':
             self = SchemaPartitionScheme.__new__(SchemaPartitionScheme)
-        elif typ == 'hive_partition_scheme':
+        elif typ == 'hive':
             self = HivePartitionScheme.__new__(HivePartitionScheme)
         else:
             raise TypeError(typ)
@@ -121,7 +120,7 @@ cdef class PartitionScheme:
 cdef class DefaultPartitionScheme(PartitionScheme):
 
     cdef:
-        CDefaultPartitionScheme* default_scheme  # hmmm...
+        CDefaultPartitionScheme* default_scheme
 
     def __init__(self):
         cdef shared_ptr[CDefaultPartitionScheme] scheme
@@ -332,12 +331,12 @@ cdef class DataSource:
     cdef wrap(shared_ptr[CDataSource]& sp):
         cdef DataSource self
 
-        typ = frombytes(sp.get().type())
-        if typ == 'tree_data_source':
+        typ = frombytes(sp.get().type_name())
+        if typ == 'tree':
             self = TreeDataSource.__new__(TreeDataSource)
-        elif typ == 'simple_data_source':
+        elif typ == 'simple':
             self = SimpleDataSource.__new__(SimpleDataSource)
-        elif typ == 'filesystem_data_source':
+        elif typ == 'filesystem':
             self = FileSystemDataSource.__new__(FileSystemDataSource)
         else:
             raise TypeError(typ)

python/pyarrow/_fs.pyx

@@ -194,7 +194,7 @@ cdef class FileSystem:
     cdef wrap(shared_ptr[CFileSystem]& sp):
         cdef FileSystem self
 
-        typ = frombytes(sp.get().type())
+        typ = frombytes(sp.get().type_name())
         if typ == 'local':
             self = LocalFileSystem.__new__(LocalFileSystem)
         elif typ == 'mock':
@@ -202,8 +202,11 @@ cdef class FileSystem:
         elif typ == 'subtree':
             self = SubTreeFileSystem.__new__(SubTreeFileSystem)
         elif typ == 's3':
-            from pyarrow.s3fs import S3FileSystem
+            from pyarrow._s3fs import S3FileSystem
             self = S3FileSystem.__new__(S3FileSystem)
+        elif typ == 'hdfs':
+            from pyarrow._hdfs import HadoopFileSystem
+            self = HadoopFileSystem.__new__(HadoopFileSystem)
         else:
             raise TypeError('Cannot wrap FileSystem pointer')
 

python/pyarrow/includes/libarrow_dataset.pxd

@@ -183,7 +183,6 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
     cdef cppclass CDataFragment "arrow::dataset::DataFragment":
         CResult[CScanTaskIterator] Scan(CScanContextPtr context)
         c_bool splittable()
-        c_string type()
         CScanOptionsPtr scan_options()
 
     ctypedef shared_ptr[CDataFragment] CDataFragmentPtr \
@@ -201,7 +200,7 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
     cdef cppclass CDataSource "arrow::dataset::DataSource":
         CDataFragmentIterator GetFragments(CScanOptionsPtr options)
         const CExpressionPtr& partition_expression()
-        c_string type()
+        c_string type_name()
 
     cdef cppclass CSimpleDataSource "arrow::dataset::SimpleDataSource"(
             CDataSource):
@@ -311,7 +310,7 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil:
         c_bool splittable()
 
     cdef cppclass CPartitionScheme "arrow::dataset::PartitionScheme":
-        c_string name() const
+        c_string type_name() const
         CResult[CExpressionPtr] Parse(const c_string& path) const
 
     ctypedef shared_ptr[CPartitionScheme] CPartitionSchemePtr \

python/pyarrow/includes/libarrow_fs.pxd

@@ -59,7 +59,7 @@ cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil:
         c_bool recursive
 
     cdef cppclass CFileSystem "arrow::fs::FileSystem":
-        c_string type() const
+        c_string type_name() const
         CResult[CFileStats] GetTargetStats(const c_string& path)
         CResult[vector[CFileStats]] GetTargetStats(
             const vector[c_string]& paths)