use pathlib in orc and parquet tests; move parquet test files to its own directory

python/pyarrow/tests/conftest.py

@@ -15,7 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pytest import skip, mark
+import pytest
+import pathlib
 
 
 groups = [
@@ -84,7 +85,7 @@ def pytest_addoption(parser):
 
 def pytest_collection_modifyitems(config, items):
     if not config.getoption('--runslow'):
-        skip_slow = mark.skip(reason='need --runslow option to run')
+        skip_slow = pytest.mark.skip(reason='need --runslow option to run')
 
         for item in items:
             if 'slow' in item.keywords:
@@ -104,7 +105,7 @@ def pytest_runtest_setup(item):
         elif getattr(item.obj, group, None):
             if (item.config.getoption(disable_flag) or
                     not item.config.getoption(flag)):
-                skip('{0} NOT enabled'.format(flag))
+                pytest.skip('{0} NOT enabled'.format(flag))
 
     if only_set:
         skip_item = True
@@ -115,4 +116,9 @@ def pytest_runtest_setup(item):
                 skip_item = False
 
         if skip_item:
-            skip('Only running some groups with only flags')
+            pytest.skip('Only running some groups with only flags')
+
+
+@pytest.fixture(scope='session')
+def datadir():
+    return pathlib.Path(__file__).parent / 'data'

python/pyarrow/tests/test_orc.py

@@ -15,32 +15,25 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import datetime
-import decimal
 import os
-
-from pandas.util.testing import assert_frame_equal
-import pandas as pd
 import pytest
+import pathlib
+import decimal
+import datetime
 
+import pandas as pd
 import pyarrow as pa
 
+from pandas.util.testing import assert_frame_equal
 
 # Marks all of the tests in this module
 # Ignore these with pytest ... -m 'not orc'
 pytestmark = pytest.mark.orc
 
 
-here = os.path.abspath(os.path.dirname(__file__))
-orc_data_dir = os.path.join(here, 'data', 'orc')
-
-
-def path_for_orc_example(name):
-    return os.path.join(orc_data_dir, '%s.orc' % name)
-
-
-def path_for_json_example(name):
-    return os.path.join(orc_data_dir, '%s.jsn.gz' % name)
+@pytest.fixture(scope='module')
+def datadir(datadir):
+    return datadir / 'orc'
 
 
 def fix_example_values(actual_cols, expected_cols):
@@ -119,31 +112,29 @@ def check_example_file(orc_path, expected_df, need_fix=False):
     assert json_pos == orc_file.nrows
 
 
-@pytest.mark.parametrize('example_name', [
-    'TestOrcFile.test1',
-    'TestOrcFile.testDate1900',
-    'decimal'
+@pytest.mark.parametrize('filename', [
+    'TestOrcFile.test1.orc',
+    'TestOrcFile.testDate1900.orc',
+    'decimal.orc'
 ])
-def test_example_using_json(example_name):
+def test_example_using_json(filename, datadir):
     """
     Check a ORC file example against the equivalent JSON file, as given
     in the Apache ORC repository (the JSON file has one JSON object per
     line, corresponding to one row in the ORC file).
     """
     # Read JSON file
-    json_path = path_for_json_example(example_name)
-    table = pd.read_json(json_path, lines=True)
-
-    check_example_file(path_for_orc_example(example_name), table,
-                       need_fix=True)
+    path = datadir / filename
+    table = pd.read_json(str(path.with_suffix('.jsn.gz')), lines=True)
+    check_example_file(path, table, need_fix=True)
 
 
-def test_orcfile_empty():
+def test_orcfile_empty(datadir):
     from pyarrow import orc
-    f = orc.ORCFile(path_for_orc_example('TestOrcFile.emptyFile'))
-    table = f.read()
+
+    table = orc.ORCFile(datadir / 'TestOrcFile.emptyFile.orc').read()
     assert table.num_rows == 0
-    schema = table.schema
+
     expected_schema = pa.schema([
         ('boolean1', pa.bool_()),
         ('byte1', pa.int8()),
@@ -172,4 +163,4 @@ def test_orcfile_empty():
                 ])),
             ]))),
         ])
-    assert schema == expected_schema
+    assert table.schema == expected_schema

python/pyarrow/tests/test_parquet.py

@@ -23,24 +23,30 @@ import io
 import json
 import os
 import sys
-
 import pytest
-
-from pyarrow.compat import guid, u, BytesIO, unichar
-from pyarrow.tests import util
-from pyarrow.filesystem import LocalFileSystem
-from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
+import pathlib
 
 import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pandas.util.testing as tm
 
+from pyarrow.compat import guid, u, BytesIO, unichar
+from pyarrow.tests import util
+from pyarrow.filesystem import LocalFileSystem
+from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
+
+
 # Marks all of the tests in this module
 # Ignore these with pytest ... -m 'not parquet'
 pytestmark = pytest.mark.parquet
 
 
+@pytest.fixture(scope='module')
+def datadir(datadir):
+    return datadir / 'parquet'
+
+
 def _write_table(table, path, **kwargs):
     import pyarrow.parquet as pq
 
@@ -1802,13 +1808,11 @@ def test_read_non_existent_file(tmpdir):
         assert path in e.args[0]
 
 
-def test_read_table_doesnt_warn():
+def test_read_table_doesnt_warn(datadir):
     import pyarrow.parquet as pq
 
-    path = os.path.join(os.path.dirname(__file__), 'data', 'v0.7.1.parquet')
-
     with pytest.warns(None) as record:
-        pq.read_table(path)
+        pq.read_table(datadir / 'v0.7.1.parquet')
 
     assert len(record) == 0
 
@@ -1972,7 +1976,7 @@ def test_parquet_nested_convenience(tmpdir):
     tm.assert_frame_equal(read.to_pandas(), df)
 
 
-def test_backwards_compatible_index_naming():
+def test_backwards_compatible_index_naming(datadir):
     expected_string = b"""\
 carat        cut  color  clarity  depth  table  price     x     y     z
  0.23      Ideal      E      SI2   61.5   55.0    326  3.95  3.98  2.43
@@ -1988,13 +1992,12 @@ carat        cut  color  clarity  depth  table  price     x     y     z
     expected = pd.read_csv(
         io.BytesIO(expected_string), sep=r'\s{2,}', index_col=None, header=0
     )
-    path = os.path.join(os.path.dirname(__file__), 'data', 'v0.7.1.parquet')
-    t = _read_table(path)
-    result = t.to_pandas()
+    table = _read_table(datadir / 'v0.7.1.parquet')
+    result = table.to_pandas()
     tm.assert_frame_equal(result, expected)
 
 
-def test_backwards_compatible_index_multi_level_named():
+def test_backwards_compatible_index_multi_level_named(datadir):
     expected_string = b"""\
 carat        cut  color  clarity  depth  table  price     x     y     z
  0.23      Ideal      E      SI2   61.5   55.0    326  3.95  3.98  2.43
@@ -2011,15 +2014,13 @@ carat        cut  color  clarity  depth  table  price     x     y     z
         io.BytesIO(expected_string),
         sep=r'\s{2,}', index_col=['cut', 'color', 'clarity'], header=0
     ).sort_index()
-    path = os.path.join(
-        os.path.dirname(__file__), 'data', 'v0.7.1.all-named-index.parquet'
-    )
-    t = _read_table(path)
-    result = t.to_pandas()
+
+    table = _read_table(datadir / 'v0.7.1.all-named-index.parquet')
+    result = table.to_pandas()
     tm.assert_frame_equal(result, expected)
 
 
-def test_backwards_compatible_index_multi_level_some_named():
+def test_backwards_compatible_index_multi_level_some_named(datadir):
     expected_string = b"""\
 carat        cut  color  clarity  depth  table  price     x     y     z
  0.23      Ideal      E      SI2   61.5   55.0    326  3.95  3.98  2.43
@@ -2037,15 +2038,13 @@ carat        cut  color  clarity  depth  table  price     x     y     z
         sep=r'\s{2,}', index_col=['cut', 'color', 'clarity'], header=0
     ).sort_index()
     expected.index = expected.index.set_names(['cut', None, 'clarity'])
-    path = os.path.join(
-        os.path.dirname(__file__), 'data', 'v0.7.1.some-named-index.parquet'
-    )
-    t = _read_table(path)
-    result = t.to_pandas()
+
+    table = _read_table(datadir / 'v0.7.1.some-named-index.parquet')
+    result = table.to_pandas()
     tm.assert_frame_equal(result, expected)
 
 
-def test_backwards_compatible_column_metadata_handling():
+def test_backwards_compatible_column_metadata_handling(datadir):
     expected = pd.DataFrame(
         {'a': [1, 2, 3], 'b': [.1, .2, .3],
          'c': pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')})
@@ -2054,16 +2053,13 @@ def test_backwards_compatible_column_metadata_handling():
          pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')],
         names=['index', None])
 
-    path = os.path.join(
-        os.path.dirname(__file__), 'data',
-        'v0.7.1.column-metadata-handling.parquet'
-    )
-    t = _read_table(path)
-    result = t.to_pandas()
+    path = datadir / 'v0.7.1.column-metadata-handling.parquet'
+    table = _read_table(path)
+    result = table.to_pandas()
     tm.assert_frame_equal(result, expected)
 
-    t = _read_table(path, columns=['a'])
-    result = t.to_pandas()
+    table = _read_table(path, columns=['a'])
+    result = table.to_pandas()
     tm.assert_frame_equal(result, expected[['a']].reset_index(drop=True))
 
 
@@ -2071,7 +2067,6 @@ def test_decimal_roundtrip(tmpdir):
     num_values = 10
 
     columns = {}
-
     for precision in range(1, 39):
         for scale in range(0, precision + 1):
             with util.random_seed(0):
@@ -2086,8 +2081,8 @@ def test_decimal_roundtrip(tmpdir):
     expected = pd.DataFrame(columns)
     filename = tmpdir.join('decimals.parquet')
     string_filename = str(filename)
-    t = pa.Table.from_pandas(expected)
-    _write_table(t, string_filename)
+    table = pa.Table.from_pandas(expected)
+    _write_table(table, string_filename)
     result_table = _read_table(string_filename)
     result = result_table.to_pandas()
     tm.assert_frame_equal(result, expected)

python/requirements.txt

@@ -1,5 +1,6 @@
+six
 pytest
 cloudpickle>=0.4.0
 numpy>=1.10.0
-six
-futures ; python_version < "3"
+futures; python_version < "3"
+pathlib2; python_version < "3.4"

python/setup.py

@@ -557,7 +557,7 @@ setup(
                      },
     setup_requires=['setuptools_scm', 'cython >= 0.27'] + setup_requires,
     install_requires=install_requires,
-    tests_require=['pytest', 'pandas'],
+    tests_require=['pytest', 'pandas', 'pathlib2; python_version < "3.4"'],
     description="Python library for Apache Arrow",
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -567,7 +567,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
-        ],
+    ],
     license='Apache License, Version 2.0',
     maintainer="Apache Arrow Developers",
     maintainer_email="dev@arrow.apache.org",