mirrors/arrow
0
0 Fork 0
mirror of https://github.com/apache/arrow.git synced 2026-10-28 16:47:19 +08:00
Code Activity

GH-48473: [CI][Python] Require numpy 2.0 (#50769)

Browse Source 
### Rationale for this change
Require Numpy 2.0 to support newer features, allow newer numpy API/ABI

### What changes are included in this PR?

- Numpy 2.0+ requirement
- Pandas 2.2.2+ requirement (first wheel compatible with numpy 2.0)
- Pyodide 0.27.1 usage (numpy 2.0 api)

### Are there any user-facing changes?
Yes, in contrast to pyarrow 25, after merging this PR pyarrow 26 can't be used with Numpy 1.x or pandas 2.0-2.2.1 

**This PR includes breaking changes to public APIs.**

This PR was created using GPT-5.6-Sol-xhigh, every line read & reviewed by me.
* GitHub Issue: #48473

Lead-authored-by: Adam Lippai <adam@rigo.sk>
Co-authored-by: Ádám Lippai <adam@rigo.sk>
Co-authored-by: Raúl Cumplido <raulcumplido@gmail.com>
Co-authored-by: Joris Van den Bossche <jorisvandenbossche@gmail.com>
Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>
This commit is contained in:
Ádám LippaiRaúl CumplidoJoris Van den Bossche
 authored and GitHub committed 2026-08-21 19:16:28 +02:00
1 parent ba5ca205d1
commit b60d43701b
32 files changed
+108 -240
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/python.yml
+5 -5
View File
@@ -67,7 +67,7 @@ jobs:
name:
- conda-python-docs
- conda-python-3.12-nopandas
- conda-python-3.11-pandas-2.0.3
- conda-python-3.11-pandas-2.2.2-numpy-2.0.2
- conda-python-3.14-pandas-latest
- conda-python-3.13-no-numpy
include:
@@ -79,12 +79,12 @@ jobs:
image: conda-python
title: AMD64 Conda Python 3.12 Without Pandas
python: "3.12"
- name: conda-python-3.11-pandas-2.0.3
- name: conda-python-3.11-pandas-2.2.2-numpy-2.0.2
image: conda-python-pandas
title: AMD64 Conda Python 3.11 Pandas 2.0.3
title: AMD64 Conda Python 3.11 Pandas 2.2.2 NumPy 2.0.2
python: "3.11"
pandas: "2.0.3"
numpy: "1.23.2"
pandas: "2.2.2"
numpy: "2.0.2"
- name: conda-python-3.14-pandas-latest
image: conda-python-pandas
title: AMD64 Conda Python 3.14 Pandas latest
ci/conda_env_python.txt
+1 -1
View File
@@ -25,7 +25,7 @@ cloudpickle
fsspec
hypothesis
libcst>=1.8.6
numpy>=1.16.6
numpy>=2.0
pytest
pytest-faulthandler
s3fs>=2023.10.0
ci/docker/conda-python-emscripten.dockerfile
+1 -1
View File
@@ -22,7 +22,7 @@ ARG python="3.12"
FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python}
ARG selenium_version="4.41.0"
ARG pyodide_version="0.26.0"
ARG pyodide_version="0.27.1"
ARG chrome_version="latest"
ARG required_python_min="(3,12)"
# fail if python version < 3.12
compose.yaml
+1 -1
View File
@@ -981,7 +981,7 @@ services:
arch_short: ${ARCH_SHORT}
clang_tools: ${CLANG_TOOLS}
llvm: ${LLVM}
pyodide_version: "0.26.0"
pyodide_version: "0.27.1"
chrome_version: "latest"
selenium_version: "4.41.0"
required_python_min: "(3,12)"
cpp/src/arrow/util/float16_test.cc
+2 -2
View File
@@ -109,7 +109,7 @@ class Float16ConversionTest : public ::testing::Test {
template <>
void Float16ConversionTest<float>::TestRoundTrip() {
// Expected values were also manually validated with numpy-1.24.3
// Expected values were also manually validated with NumPy 1.24.3
const RoundTripTestCase test_cases[] = {
// +/-0.0f
{F32(0x80000000u), 0b1000000000000000u, -0.0f},
@@ -148,7 +148,7 @@ void Float16ConversionTest<float>::TestRoundTrip() {
template <>
void Float16ConversionTest<double>::TestRoundTrip() {
// Expected values were also manually validated with numpy-1.24.3
// Expected values were also manually validated with NumPy 1.24.3
const RoundTripTestCase test_cases[] = {
// +/-0.0
{F64(0x8000000000000000u), 0b1000000000000000u, -0.0},
dev/tasks/tasks.yml
+2 -2
View File
@@ -730,9 +730,9 @@ tasks:
############################## Integration tests ############################
{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "2.0.3", "1.23.2", True),
{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "2.2.2", "2.0.2", True),
("3.12", "latest", "latest", False),
("3.13", "latest", "1.26.2", False),
("3.13", "latest", "2.1.3", False),
("3.13", "latest", "latest", False),
("3.14", "nightly", "nightly", False),
("3.14", "upstream_devel", "nightly", False)] %}
docs/source/python/extending_types.rst
+2 -2
View File
@@ -328,9 +328,9 @@ a built-in :class:`ExtensionArray` object. Nevertheless, one could want to subcl
type. Arrow allows to do so by adding a special method ``__arrow_ext_class__`` to the
definition of the extension type.
For instance, let us consider the example from the `Numpy Quickstart <https://docs.scipy.org/doc/numpy-1.13.0/user/quickstart.html>`_ of points in 3D space.
For instance, let us consider the example from the `NumPy Quickstart <https://numpy.org/doc/stable/user/quickstart.html>`_ of points in 3D space.
We can store these as a fixed-size list, where we wish to be able to extract
the data as a 2-D Numpy array ``(N, 3)`` without any copy:
the data as a 2-D NumPy array ``(N, 3)`` without any copy:
.. code-block:: python
docs/source/python/install.rst
+2 -2
View File
@@ -73,8 +73,8 @@ Dependencies
Optional dependencies
* **NumPy 1.23.2** or higher.
* **pandas 2.0.3** or higher,
* **NumPy 2.0** or higher.
* **pandas 2.2.2** or higher,
* **cffi**.
Additional packages PyArrow is compatible with are :ref:`fsspec <filesystem-fsspec>`
python/CMakeLists.txt
+5 -10
View File
@@ -167,16 +167,7 @@ if($ENV{PYODIDE})
set(Python3_INCLUDE_DIR $ENV{PYTHONINCLUDE})
set(Python3_LIBRARY $ENV{CPYTHONLIB})
set(Python3_EXECUTABLE)
execute_process(COMMAND ${Python3_EXECUTABLE} -c
"import numpy; print(numpy.__version__)"
OUTPUT_VARIABLE PYODIDE_NUMPY_VERSION
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REGEX MATCH "^([0-9]+)" PYODIDE_NUMPY_MAJOR_VERSION ${PYODIDE_NUMPY_VERSION})
if(PYODIDE_NUMPY_MAJOR_VERSION GREATER_EQUAL 2)
set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/_core/include)
else()
set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/core/include)
endif()
set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/_core/include)
set(ENV{_PYTHON_SYSCONFIGDATA_NAME} $ENV{SYSCONFIG_NAME})
# we set the c and cxx compiler manually to bypass pywasmcross
# which is pyodide's way of messing with C++ build parameters.
@@ -278,6 +269,10 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
# Python and Numpy libraries
find_package(Python3Alt REQUIRED)
if(Python3_NumPy_VERSION AND Python3_NumPy_VERSION VERSION_LESS "2.0")
message(FATAL_ERROR "PyArrow requires NumPy 2.0 or newer, found ${Python3_NumPy_VERSION}"
)
endif()
message(STATUS "Found NumPy version: ${Python3_NumPy_VERSION}")
message(STATUS "NumPy include dir: ${NUMPY_INCLUDE_DIRS}")
python/benchmarks/common.py
+1 -1
View File
@@ -149,7 +149,7 @@ class BuiltinsGenerator(object):
Generate a list of Python bools with *none_prob* probability of
an entry being None.
"""
# Make sure we get Python bools, not np.bool_
# Make sure we get Python bools, not np.bool
data = [bool(x >= 0.5) for x in self.rnd.uniform(0.0, 1.0, n)]
assert len(data) == n
self.sprinkle_nones(data, none_prob)
python/pyarrow/array.pxi
+14 -19
View File
@@ -342,7 +342,7 @@ def array(object obj, type=None, mask=None, size=None, from_pandas=None,
values = values.data
if mask is not None:
if mask.dtype != np.bool_:
if mask.dtype != np.bool:
raise TypeError("Mask must be boolean dtype")
if mask.ndim != 1:
raise ValueError("Mask must be 1D array")
@@ -945,14 +945,12 @@ cdef class _PandasConvertible(_Weakrefable):
Cast integers with nulls to objects
date_as_object : bool, default True
Cast dates to objects. If False, convert to datetime64 dtype with
the equivalent time unit (if supported). Note: in pandas version
< 2.0, only datetime64[ns] conversion is supported.
the equivalent time unit (if supported).
timestamp_as_object : bool, default False
Cast non-nanosecond timestamps (np.datetime64) to objects. This is
useful in pandas version 1.x if you have timestamps that don't fit
in the normal date range of nanosecond timestamps (1678 CE-2262 CE).
Non-nanosecond timestamps are supported in pandas version 2.0.
If False, all timestamps are converted to datetime64 dtype.
Cast non-nanosecond timestamps (np.datetime64) to objects. This can
be useful when Python datetime objects are required, such as for
compatibility with code expecting object dtype. If False, all
timestamps are converted to datetime64 dtype.
use_threads : bool, default True
Whether to parallelize the conversion using multiple threads.
deduplicate_objects : bool, default True
@@ -963,9 +961,8 @@ cdef class _PandasConvertible(_Weakrefable):
DataFrame index, if present
safe : bool, default True
For certain data types, a cast is needed in order to store the
data in a pandas DataFrame or Series (e.g. timestamps are always
stored as nanoseconds in pandas). This option controls whether it
is a safe cast or not.
data in a pandas DataFrame or Series. This option controls whether
it is a safe cast or not.
split_blocks : bool, default False
If True, generate one internal "block" for each column when
creating a pandas.DataFrame from a RecordBatch or Table. While this
@@ -1002,12 +999,10 @@ cdef class _PandasConvertible(_Weakrefable):
default conversion should be used for that type. If you have
a dictionary mapping, you can pass ``dict.get`` as function.
coerce_temporal_nanoseconds : bool, default False
Only applicable to pandas version >= 2.0.
A legacy option to coerce date32, date64, duration, and timestamp
time units to nanoseconds when converting to pandas. This is the
default behavior in pandas version 1.x. Set this option to True if
you'd like to use this coercion when using pandas version >= 2.0
for backwards compatibility (not recommended otherwise).
time units to nanoseconds when converting to pandas. Set this
option only if nanosecond coercion is required for compatibility
with older application behavior.
Returns
-------
@@ -5130,7 +5125,7 @@ cdef class Bool8Array(ExtensionArray):
"""
if not writable:
try:
return self.storage.to_numpy().view(np.bool_)
return self.storage.to_numpy().view(np.bool)
except ArrowInvalid as e:
if zero_copy_only:
raise e
@@ -5171,7 +5166,7 @@ cdef class Bool8Array(ExtensionArray):
--------
>>> import pyarrow as pa
>>> import numpy as np
>>> arr = np.array([True, False, True], dtype=np.bool_)
>>> arr = np.array([True, False, True], dtype=np.bool)
>>> pa.Bool8Array.from_numpy(arr)
<pyarrow.lib.Bool8Array object at ...>
[
@@ -5184,7 +5179,7 @@ cdef class Bool8Array(ExtensionArray):
if obj.ndim != 1:
raise ValueError(f"Cannot convert {obj.ndim}-D array to bool8 array")
if obj.dtype not in [np.bool_, np.int8]:
if obj.dtype not in [np.bool, np.int8]:
raise TypeError(f"Array dtype {obj.dtype} incompatible with bool8 storage")
storage_arr = array(obj.view(np.int8), type=int8())
python/pyarrow/feather.py
-5
View File
@@ -150,11 +150,6 @@ def write_feather(df, dest, compression=None, compression_level=None,
DeprecationWarning,
stacklevel=2
)
if _pandas_api.have_pandas:
if (_pandas_api.has_sparse and
isinstance(df, _pandas_api.pd.SparseDataFrame)):
df = df.to_dense()
if _pandas_api.is_data_frame(df):
# Feather v1 creates a new column in the resultant Table to
# store index information if index type is not RangeIndex
python/pyarrow/lib.pyx
+4
View File
@@ -38,6 +38,10 @@ cimport cpython as cp
# Initialize NumPy C API only if numpy was able to be imported
if np is not None:
if int(np.__version__.partition('.')[0]) < 2:
raise ImportError(
f"pyarrow requires NumPy 2.0 or newer, found {np.__version__}"
)
arrow_init_numpy()
# Initialize PyArrow C++ API
python/pyarrow/pandas-shim.pxi
+9 -41
View File
@@ -35,10 +35,8 @@ cdef class _PandasAPIShim(object):
object _pd, _types_api, _compat_module
object _data_frame, _index, _series, _categorical_type
object _datetimetz_type, _extension_array, _extension_dtype
object _array_like_types, _is_extension_array_dtype, _lock
bint has_sparse
bint _pd024
bint _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict
object _array_like_types, _lock
bint _is_ge_v23, _is_ge_v3, _is_ge_v3_strict
def __init__(self):
self._lock = Lock()
@@ -62,23 +60,22 @@ cdef class _PandasAPIShim(object):
self._version = pd.__version__
self._loose_version = Version(pd.__version__)
if self._loose_version < Version('2.0.3'):
if self._loose_version < Version('2.2.2'):
self._have_pandas = False
if raise_:
raise ImportError(
f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is "
f"pyarrow requires pandas 2.2.2 or above, pandas {self._version} is "
"installed"
)
else:
warnings.warn(
f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is "
f"pyarrow requires pandas 2.2.2 or above, pandas {self._version} is "
"installed. Therefore, pandas-specific integration is not "
"used.",
stacklevel=2
)
return
self._is_ge_v21 = self._loose_version >= Version('2.1.0')
self._is_ge_v23 = self._loose_version >= Version('2.3.0.dev0')
self._is_ge_v3 = self._loose_version >= Version('3.0.0.dev0')
self._is_ge_v3_strict = self._loose_version >= Version('3.0.0')
@@ -93,12 +90,9 @@ cdef class _PandasAPIShim(object):
self._series, self._index, self._categorical_type,
self._extension_array)
self._extension_dtype = pd.api.extensions.ExtensionDtype
self._is_extension_array_dtype = (
pd.api.types.is_extension_array_dtype)
self._types_api = pd.api.types
self._datetimetz_type = pd.api.types.DatetimeTZDtype
self._have_pandas = True
self.has_sparse = False
cdef inline _check_import(self, bint raise_=True):
if not self._tried_importing_pandas:
@@ -142,17 +136,11 @@ cdef class _PandasAPIShim(object):
cpdef infer_dtype(self, obj):
self._check_import()
try:
return self._types_api.infer_dtype(obj, skipna=False)
except AttributeError:
return self._pd.lib.infer_dtype(obj)
return self._types_api.infer_dtype(obj, skipna=False)
cpdef pandas_dtype(self, dtype):
self._check_import()
try:
return self._types_api.pandas_dtype(dtype)
except AttributeError:
return None
return self._types_api.pandas_dtype(dtype)
@property
def loose_version(self):
@@ -164,10 +152,6 @@ cdef class _PandasAPIShim(object):
self._check_import()
return self._version
def is_ge_v21(self):
self._check_import()
return self._is_ge_v21
def is_ge_v23(self):
self._check_import()
return self._is_ge_v23
@@ -183,12 +167,7 @@ cdef class _PandasAPIShim(object):
def uses_string_dtype(self):
if self.is_ge_v3_strict():
return True
try:
if self.is_ge_v23() and self.pd.options.future.infer_string:
return True
except:
pass
return False
return self.is_ge_v23() and self.pd.options.future.infer_string
@property
def categorical_type(self):
@@ -223,10 +202,7 @@ cdef class _PandasAPIShim(object):
cpdef is_extension_array_dtype(self, obj):
self._check_import()
if self._is_extension_array_dtype:
return self._is_extension_array_dtype(obj)
else:
return False
return self._types_api.is_extension_array_dtype(obj)
cpdef is_sparse(self, obj):
if self._have_pandas_internal():
@@ -265,13 +241,5 @@ cdef class _PandasAPIShim(object):
return obj.array
return obj.values
def get_rangeindex_attribute(self, level, name):
# public start/stop/step attributes added in pandas 0.25.0
self._check_import()
if hasattr(level, name):
return getattr(level, name)
return getattr(level, '_' + name)
cdef _PandasAPIShim pandas_api = _PandasAPIShim()
_pandas_api = pandas_api
python/pyarrow/pandas_compat.py
+9 -25
View File
@@ -93,7 +93,7 @@ def get_numpy_logical_type_map():
global _numpy_logical_type_map
if not _numpy_logical_type_map:
_numpy_logical_type_map.update({
np.bool_: 'bool',
np.bool: 'bool',
np.int8: 'int8',
np.int16: 'int16',
np.int32: 'int32',
@@ -277,7 +277,7 @@ def construct_metadata(columns_to_convert, df, column_names, index_levels,
else:
index_descriptors = index_column_metadata = column_indexes = []
attributes = df.attrs if hasattr(df, "attrs") else {}
attributes = df.attrs
try:
json.dumps(attributes)
@@ -537,13 +537,12 @@ def _level_name(name):
def _get_range_index_descriptor(level):
# public start/stop/step attributes added in pandas 0.25.0
return {
'kind': 'range',
'name': _level_name(level.name),
'start': _pandas_api.get_rangeindex_attribute(level, 'start'),
'stop': _pandas_api.get_rangeindex_attribute(level, 'stop'),
'step': _pandas_api.get_rangeindex_attribute(level, 'step')
'start': level.start,
'stop': level.stop,
'step': level.step
}
@@ -759,17 +758,9 @@ def _reconstruct_block(item, columns=None, extension_columns=None, return_block=
elif 'timezone' in item:
unit, _ = np.datetime_data(block_arr.dtype)
dtype = make_datetimetz(unit, item['timezone'])
if _pandas_api.is_ge_v21():
arr = _pandas_api.pd.array(
block_arr.view("int64"), dtype=dtype, copy=False
)
else:
arr = block_arr
if return_block:
block = _int.make_block(block_arr, placement=placement,
klass=_int.DatetimeTZBlock,
dtype=dtype)
return block
arr = _pandas_api.pd.array(
block_arr.view("int64"), dtype=dtype, copy=False
)
elif 'py_array' in item:
# create ExtensionBlock
arr = item['py_array']
@@ -846,10 +837,7 @@ def table_to_dataframe(
]
axes = [columns, index]
mgr = BlockManager(blocks, axes)
if _pandas_api.is_ge_v21():
df = DataFrame._from_mgr(mgr, mgr.axes)
else:
df = DataFrame(mgr)
df = DataFrame._from_mgr(mgr, mgr.axes)
df.attrs = attributes
@@ -884,10 +872,6 @@ def _get_extension_dtypes(table, columns_metadata, types_mapper, options, catego
ext_columns = {}
# older pandas version that does not yet support extension dtypes
if _pandas_api.extension_dtype is None:
return ext_columns
# use the specified mapping of built-in arrow types to pandas dtypes
if types_mapper:
for field in table.schema:
python/pyarrow/src/arrow/python/numpy_interop.h
+8 -17
View File
@@ -19,16 +19,14 @@
#include "arrow/python/platform.h" // IWYU pragma: export
// Require the NumPy 2.0 C API and hide deprecated APIs.
#define NPY_TARGET_VERSION NPY_2_0_API_VERSION
#define NPY_NO_DEPRECATED_API NPY_2_0_API_VERSION
#include <numpy/numpyconfig.h> // IWYU pragma: export
// Don't use the deprecated Numpy functions
#ifdef NPY_1_7_API_VERSION
# define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#else
# define NPY_ARRAY_NOTSWAPPED NPY_NOTSWAPPED
# define NPY_ARRAY_ALIGNED NPY_ALIGNED
# define NPY_ARRAY_WRITEABLE NPY_WRITEABLE
# define NPY_ARRAY_UPDATEIFCOPY NPY_UPDATEIFCOPY
#if NPY_ABI_VERSION < 0x02000000
# error "PyArrow requires NumPy 2.0 or newer"
#endif
// This is required to be able to access the NumPy C API properly in C++ files
@@ -67,20 +65,13 @@
# define NPY_INT32_IS_INT 0
#endif
// Backported NumPy 2 API (can be removed if numpy 2 is required)
#if NPY_ABI_VERSION < 0x02000000
# define PyDataType_ELSIZE(descr) ((descr)->elsize)
# define PyDataType_C_METADATA(descr) ((descr)->c_metadata)
# define PyDataType_FIELDS(descr) ((descr)->fields)
#endif
namespace arrow {
namespace py {
inline int import_numpy() {
#ifdef NUMPY_IMPORT_ARRAY
import_array1(-1);
import_umath1(-1);
if (PyArray_ImportNumPyAPI() < 0) return -1;
if (PyUFunc_ImportUFuncAPI() < 0) return -1;
#endif
return 0;
python/pyarrow/tests/parquet/test_pandas.py
+6 -9
View File
@@ -27,7 +27,6 @@ import pytest
import pyarrow as pa
from pyarrow.fs import LocalFileSystem, SubTreeFileSystem
from pyarrow.util import guid
from pyarrow.vendored.version import Version
try:
import pyarrow.parquet as pq
@@ -430,22 +429,20 @@ carat cut color clarity depth table price x y z
@pytest.mark.pandas
def test_backwards_compatible_column_metadata_handling(datadir):
if Version("2.2.0") <= Version(pd.__version__):
# TODO: regression in pandas
# https://github.com/pandas-dev/pandas/issues/56775
pytest.skip("Regression in pandas 2.2.0")
dates = pd.date_range(
"2017-01-01", periods=3, tz='Europe/Brussels'
).as_unit("ns")
expected = pd.DataFrame(
{'a': [1, 2, 3], 'b': [.1, .2, .3],
'c': pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')})
'c': dates})
expected.index = pd.MultiIndex.from_arrays(
[['a', 'b', 'c'],
pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')],
[['a', 'b', 'c'], dates],
names=['index', None])
path = datadir / 'v0.7.1.column-metadata-handling.parquet'
table = _read_table(path)
result = table.to_pandas()
tm.assert_frame_equal(result, expected)
tm.assert_frame_equal(result, expected, check_freq=False)
table = _read_table(
path, columns=['a'])
python/pyarrow/tests/test_array.py
+6 -16
View File
@@ -34,7 +34,6 @@ except ImportError:
import pyarrow as pa
import pyarrow.tests.strategies as past
from pyarrow.vendored.version import Version
import pyarrow.compute as pc
@@ -3800,21 +3799,12 @@ def test_numpy_array_protocol():
result = np.asarray(arr)
np.testing.assert_array_equal(result, expected)
if Version(np.__version__) < Version("2.0.0.dev0"):
# copy keyword is not strict and not passed down to __array__
result = np.array(arr, copy=False)
np.testing.assert_array_equal(result, expected)
with pytest.raises(ValueError, match="Unable to avoid a copy"):
np.array(arr, copy=False)
result = np.array(arr, dtype="float64", copy=False)
np.testing.assert_array_equal(result, expected)
else:
# starting with numpy 2.0, the copy=False keyword is assumed to be strict
with pytest.raises(ValueError, match="Unable to avoid a copy"):
np.array(arr, copy=False)
arr = pa.array([1, 2, 3])
with pytest.raises(ValueError):
np.array(arr, dtype="float64", copy=False)
arr = pa.array([1, 2, 3])
with pytest.raises(ValueError):
np.array(arr, dtype="float64", copy=False)
# copy=True -> not yet passed by numpy, so we have to call this directly to test
arr = pa.array([1, 2, 3])
@@ -4436,7 +4426,7 @@ def test_non_cpu_array():
ctx = cuda.Context(0)
data = np.arange(4, dtype=np.int32)
validity = np.array([True, False, True, False], dtype=np.bool_)
validity = np.array([True, False, True, False], dtype=np.bool)
cuda_data_buf = ctx.buffer_from_data(data)
cuda_validity_buf = ctx.buffer_from_data(validity)
arr = pa.Array.from_buffers(pa.int32(), 4, [None, cuda_data_buf])
python/pyarrow/tests/test_convert_builtin.py
+1 -1
View File
@@ -240,7 +240,7 @@ def test_sequence_boolean(seq):
@pytest.mark.numpy
@parametrize_with_sequence_types
def test_sequence_numpy_boolean(seq):
expected = [np.bool_(True), None, np.bool_(False), None]
expected = [np.bool(True), None, np.bool(False), None]
arr = pa.array(seq(expected))
assert arr.type == pa.bool_()
assert arr.to_pylist() == [True, None, False, None]
python/pyarrow/tests/test_dlpack.py
+3 -15
View File
@@ -92,11 +92,6 @@ def check_bytes_allocated(f):
]
)
def test_dlpack(value_type, np_type_str):
if Version(np.__version__) < Version("1.24.0"):
pytest.skip("No dlpack support in numpy versions older than 1.22.0, "
"strict keyword in assert_array_equal added in numpy version "
"1.24.0")
expected = np.array([1, 2, 3], dtype=np.dtype(np_type_str))
arr = pa.array(expected, type=value_type)
check_dlpack_export(arr, expected)
@@ -130,11 +125,6 @@ def test_dlpack(value_type, np_type_str):
np.int8, np.int16, np.int32, np.int64,
np.float16, np.float32, np.float64,])
def test_tensor_dlpack(np_type):
if Version(np.__version__) < Version("1.24.0"):
pytest.skip("No dlpack support in numpy versions older than 1.22.0, "
"strict keyword in assert_array_equal added in numpy version "
"1.24.0")
arr = np.array([1, 2, 3, 4, 5, 6, 1, 1])
expected = np.array(arr, dtype=np_type).reshape((2, 2, 2), order='C')
t = pa.Tensor.from_numpy(expected)
@@ -227,8 +217,9 @@ def test_dlpack_versioned_roundtrip(obj):
@check_bytes_allocated
def test_dlpack_copy_is_writeable():
if Version(np.__version__) < Version("2.1.0"):
pytest.skip("Read-only DLPack flag requires numpy 2.1.0 or later")
# NumPy did not set the writeable flag on DLPack imports before 2.2.5.
if Version(np.__version__) < Version("2.2.5"):
pytest.skip("Writable DLPack imports require numpy 2.2.5 or later")
arr = pa.array([1, 2, 3], type=pa.int32())
@@ -244,9 +235,6 @@ def test_dlpack_copy_is_writeable():
def test_dlpack_not_supported():
if Version(np.__version__) < Version("1.22.0"):
pytest.skip("No dlpack support in numpy versions older than 1.22.0.")
arr = pa.array([1, None, 3])
with pytest.raises(TypeError, match="Can only use DLPack "
"on arrays with no nulls."):
python/pyarrow/tests/test_extension_type.py
+7 -12
View File
@@ -31,7 +31,6 @@ except ImportError:
np = None
import pyarrow as pa
from pyarrow.vendored.version import Version
@contextlib.contextmanager
@@ -1965,13 +1964,9 @@ def test_extension_to_pandas_storage_type(registered_period_type):
assert result["ext"].dtype == pandas_dtype
import pandas as pd
# Skip tests for 2.0.x, See: GH-35821
if (
Version(pd.__version__) >= Version("2.1.0")
):
# Check the usage of types_mapper
result = table.to_pandas(types_mapper=pd.ArrowDtype)
assert isinstance(result["ext"].dtype, pd.ArrowDtype)
# Check the usage of types_mapper
result = table.to_pandas(types_mapper=pd.ArrowDtype)
assert isinstance(result["ext"].dtype, pd.ArrowDtype)
def test_tensor_type_is_picklable(pickle_module):
@@ -2127,7 +2122,7 @@ def test_bool8_to_numpy_conversion():
)
# zero-copy possible with non-null array
np_arr_no_nulls = np.array([True, False, True, True], dtype=np.bool_)
np_arr_no_nulls = np.array([True, False, True, True], dtype=np.bool)
arr_no_nulls = pa.ExtensionArray.from_storage(
pa.bool8(),
pa.array([-1, 0, 1, 2], pa.int8()),
@@ -2149,7 +2144,7 @@ def test_bool8_to_numpy_conversion():
@pytest.mark.numpy
def test_bool8_from_numpy_conversion():
np_arr_no_nulls = np.array([True, False, True, True], dtype=np.bool_)
np_arr_no_nulls = np.array([True, False, True, True], dtype=np.bool)
canonical_bool8_arr_no_nulls = pa.ExtensionArray.from_storage(
pa.bool8(),
pa.array([1, 0, 1, 1], pa.int8()),
@@ -2167,14 +2162,14 @@ def test_bool8_from_numpy_conversion():
match="Cannot convert 2-D array to bool8 array",
):
pa.Bool8Array.from_numpy(
np.array([[True, False], [False, True]], dtype=np.bool_),
np.array([[True, False], [False, True]], dtype=np.bool),
)
with pytest.raises(
ValueError,
match="Cannot convert 0-D array to bool8 array",
):
pa.Bool8Array.from_numpy(np.bool_())
pa.Bool8Array.from_numpy(np.bool())
# must use compatible storage type
with pytest.raises(
python/pyarrow/tests/test_feather.py
-14
View File
@@ -588,20 +588,6 @@ def test_filelike_objects(version):
assert_frame_equal(result, df)
@pytest.mark.pandas
@pytest.mark.filterwarnings("ignore:Sparse:FutureWarning")
@pytest.mark.filterwarnings("ignore:DataFrame.to_sparse:FutureWarning")
def test_sparse_dataframe(version):
if not pa.pandas_compat._pandas_api.has_sparse:
pytest.skip("version of pandas does not support SparseDataFrame")
# GH #221
data = {'A': [0, 1, 2],
'B': [1, 0, 1]}
df = pd.DataFrame(data).to_sparse(fill_value=1)
expected = df.to_dense()
_check_pandas_roundtrip(df, expected, version=version)
@pytest.mark.pandas
def test_duplicate_columns_pandas():
python/pyarrow/tests/test_io.py
+2 -6
View File
@@ -917,16 +917,12 @@ def test_compression_level(compression):
with pytest.raises(ValueError):
codec.decompress(compressed_bytes)
# The ability to set a seed this way is not present on older versions of
# numpy (currently in our python 3.6 CI build). Some inputs might just
# happen to compress the same between the two levels so using seeded
# random numbers is necessary to help get more reliable results
# Some inputs might just happen to compress the same between the two levels,
# so using seeded random numbers makes the results more reliable.
#
# The goal of this part is to ensure the compression_level is being
# passed down to the C++ layer, not to verify the compression algs
# themselves
if not hasattr(np.random, 'default_rng'):
pytest.skip('Requires newer version of numpy')
rng = np.random.default_rng(seed=42)
values = rng.integers(0, 100, 1000)
arr = pa.array(values)
python/pyarrow/tests/test_pandas.py
+7 -11
View File
@@ -254,13 +254,13 @@ class TestConvertMetadata:
result = table.to_pandas()
tm.assert_frame_equal(result, df)
assert isinstance(result.index, pd.RangeIndex)
assert _pandas_api.get_rangeindex_attribute(result.index, 'step') == 2
assert result.index.step == 2
assert result.index.name == index_name
result2 = table_no_index_name.to_pandas()
tm.assert_frame_equal(result2, df2)
assert isinstance(result2.index, pd.RangeIndex)
assert _pandas_api.get_rangeindex_attribute(result2.index, 'step') == 1
assert result2.index.step == 1
assert result2.index.name is None
def test_range_index_force_serialization(self):
@@ -1686,10 +1686,6 @@ class TestConvertDateTimeLikeTypes:
expected = pd.Series([None, date(1991, 1, 1), None])
assert pa.Array.from_pandas(expected).equals(result)
@pytest.mark.skipif(
np is not None and Version('1.16.0') <= Version(
np.__version__) < Version('1.16.1'),
reason='Until numpy/numpy#12745 is resolved')
def test_fixed_offset_timezone(self):
df = pd.DataFrame({
'a': [
@@ -2783,7 +2779,7 @@ class TestConvertStructTypes:
def test_from_numpy(self):
dt = np.dtype([('x', np.int32),
(('y_title', 'y'), np.bool_)])
(('y_title', 'y'), np.bool)])
ty = pa.struct([pa.field('x', pa.int32()),
pa.field('y', pa.bool_())])
@@ -2797,7 +2793,7 @@ class TestConvertStructTypes:
{'x': 43, 'y': False}]
# With mask
arr = pa.array(data, mask=np.bool_([False, True]), type=ty)
arr = pa.array(data, mask=np.bool([False, True]), type=ty)
assert arr.to_pylist() == [{'x': 42, 'y': True}, None]
# Trivial struct type
@@ -2815,7 +2811,7 @@ class TestConvertStructTypes:
def test_from_numpy_nested(self):
# Note: an object field inside a struct
dt = np.dtype([('x', np.dtype([('xx', np.int8),
('yy', np.bool_)])),
('yy', np.bool)])),
('y', np.int16),
('z', np.object_)])
# Note: itemsize is not necessarily a multiple of sizeof(object)
@@ -2899,7 +2895,7 @@ class TestConvertStructTypes:
ty = pa.struct([pa.field('x', pa.int32()),
pa.field('y', pa.bool_())])
dt = np.dtype([('x', np.int32),
('z', np.bool_)])
('z', np.bool)])
data = np.array([], dtype=dt)
with pytest.raises(ValueError,
@@ -3859,7 +3855,7 @@ def test_array_uses_memory_pool():
# ARROW-6570
N = 10000
arr = pa.array(np.arange(N, dtype=np.int64),
mask=np.random.randint(0, 2, size=N).astype(np.bool_))
mask=np.random.randint(0, 2, size=N).astype(np.bool))
# In the case the gc is caught loading
gc.collect()
python/pyarrow/tests/test_schema.py
+1 -1
View File
@@ -51,7 +51,7 @@ def test_type_to_pandas_dtype():
M8 = np.dtype('datetime64[ms]')
cases = [
(pa.null(), np.object_),
(pa.bool_(), np.bool_),
(pa.bool_(), np.bool),
(pa.int8(), np.int8),
(pa.int16(), np.int16),
(pa.int32(), np.int32),
python/pyarrow/tests/test_table.py
+2 -10
View File
@@ -28,7 +28,6 @@ import pytest
import pyarrow as pa
import pyarrow.compute as pc
from pyarrow.interchange import from_dataframe
from pyarrow.vendored.version import Version
def test_chunked_array_basics():
@@ -3556,16 +3555,9 @@ def test_numpy_asarray(constructor):
@pytest.mark.parametrize("constructor", [pa.table, pa.record_batch])
def test_numpy_array_protocol(constructor):
table = constructor([[1, 2, 3], [4.0, 5.0, 6.0]], names=["a", "b"])
expected = np.array([[1, 4], [2, 5], [3, 6]], dtype="float64")
if Version(np.__version__) < Version("2.0.0.dev0"):
# copy keyword is not strict and not passed down to __array__
result = np.array(table, copy=False)
np.testing.assert_array_equal(result, expected)
else:
# starting with numpy 2.0, the copy=False keyword is assumed to be strict
with pytest.raises(ValueError, match="Unable to avoid a copy"):
np.array(table, copy=False)
with pytest.raises(ValueError, match="Unable to avoid a copy"):
np.array(table, copy=False)
@pytest.mark.acero
python/pyarrow/tests/test_types.py
+2 -2
View File
@@ -1350,8 +1350,8 @@ def test_is_boolean_value():
assert pa.types.is_boolean_value(True)
assert pa.types.is_boolean_value(False)
if np is not None:
assert pa.types.is_boolean_value(np.bool_(True))
assert pa.types.is_boolean_value(np.bool_(False))
assert pa.types.is_boolean_value(np.bool(True))
assert pa.types.is_boolean_value(np.bool(False))
@h.settings(suppress_health_check=(h.HealthCheck.too_slow,))
python/pyarrow/types.pxi
+2 -3
View File
@@ -31,8 +31,7 @@ import sys
import warnings
from cython import sizeof
# These are imprecise because the type (in pandas 0.x) depends on the presence
# of nulls
# These are imprecise because the type depends on the presence of nulls
cdef dict _pandas_type_map = {}
@@ -41,7 +40,7 @@ def _get_pandas_type_map():
if not _pandas_type_map:
_pandas_type_map.update({
_Type_NA: np.object_, # NaNs
_Type_BOOL: np.bool_,
_Type_BOOL: np.bool,
_Type_INT8: np.int8,
_Type_INT16: np.int16,
_Type_INT32: np.int32,
python/pyproject.toml
+1 -1
View File
@@ -22,7 +22,7 @@ requires = [
# Needed for build-time stub docstring extraction
"libcst>=1.9.0; python_version >= '3.15'",
"libcst>=1.8.6; python_version < '3.15'",
"numpy>=1.25",
"numpy>=2.0",
"setuptools_scm[toml]>=8",
]
build-backend = "scikit_build_core.build"
python/requirements-build.txt
+1 -1
View File
@@ -2,6 +2,6 @@ build
cython>=3.1
libcst>=1.9.0; python_version >= "3.15"
libcst>=1.8.6; python_version < "3.15"
numpy>=1.25
numpy>=2.0
scikit-build-core>=1.0
setuptools_scm>=8
python/requirements-wheel-test.txt
+1 -3
View File
@@ -12,9 +12,7 @@ tzdata; sys_platform == 'win32'
# version. However, there is no need to make this strictly the oldest version,
# so it can be broadened to have a single version specification across platforms.
# (`~=x.y.z` specifies a compatible release as `>=x.y.z, == x.y.*`)
numpy~=1.23.2; python_version < "3.11"
numpy~=1.23.2; python_version == "3.11"
numpy~=1.26.0; python_version == "3.12"
numpy~=2.0.0; python_version <= "3.12"
numpy~=2.1.0; python_version == "3.13"
numpy~=2.3.3; python_version == "3.14"
numpy~=2.5.2; python_version == "3.15" # TODO: 2.5.2 is for 3.15-rc1, after release, update to correct version
python/setup.cfg
-1
View File
@@ -24,7 +24,6 @@ addopts = --ignore=scripts
filterwarnings =
# https://github.com/apache/arrow/issues/49227
ignore:pyarrow.gandiva is deprecated:FutureWarning
error:The SparseDataFrame:FutureWarning
# https://github.com/apache/arrow/issues/38239
ignore:Setting custom ClientSession:DeprecationWarning
# https://github.com/apache/arrow/issues/49255