Files
Warren Deng d602395970 [torch._native] Fall back when the Triton driver is unavailable (#197695)
# Summary

Outer-product `bmm` dispatches to Triton through `torch._native`. However, driver setup can fail when Python headers, a host compiler, or another prerequisite from Triton is unavailable even if `triton` is importable. In this case, the user hits failure instead of falling back to aten implementation. This is the failure reported in #196977.

# Change

Add an optional `try_initialize_runtime` hook to native override registration. The router calls it after the predicate matches and before invoking the implementation. A `False` result continues to the next override or the captured ATen kernel.

Conditional Triton overrides use a cached `_try_initialize_driver` hook backed by `triton.runtime.driver.active`. Unconditional overrides do not accept runtime initialization hooks, and graph construction does not run them. This keeps import and nonmatching operations lazy while leaving implementation, kernel-compilation, and launch errors visible.

# Test plan

```bash
python test/python_native/test_registry.py -k runtime_initialization_falls_through -v
python test/python_native/test_registry.py -k fake_tensor_shape_inference -v
python test/python_native/test_registry.py -k unconditional_override_cond_none -v
python test/python_native/test_registry.py -k unconditional_override_rejects_runtime_initialization -v
python test/python_native/test_registry.py -k override_node_dataclass -v
python test/python_native/test_registry.py -k register_op_override_basic -v
python test/python_native/test_native_dsl_ops.py -k triton_driver_initialization_is_cached -v
python test/python_native/test_native_dsl_ops.py -k version_skip_env_var_overrides -v
python test/python_native/test_decomp.py -k export_ -v
```

All focused tests passed. Fresh-cache missing-header and invalid-compiler runs used ATen; a healthy cold start used Triton. The cached hook measured 47.8 ns, and repeated fallback measured 30.3 μs versus 15.9 μs for direct ATen.

Authored with assistance from Codex

Fixes #196977

Pull Request resolved: https://github.com/pytorch/pytorch/pull/197695
Approved by: https://github.com/slayton58
2026-09-23 07:10:51 +00:00

1398 lines
54 KiB
Python

# Owner(s): ["module: dsl-native-ops"]
import contextlib
from unittest.mock import MagicMock, patch
import torch
import torch._dynamo.trace_rules as trace_rules
import torch._native.registry as registry_module
from torch._subclasses.fake_tensor import FakeTensorMode
from torch.testing._internal.common_utils import run_tests, skipIfTorchDynamo, TestCase
class _CowStateWrapperTensor(torch.Tensor):
elem: torch.Tensor
__slots__ = ["elem"]
@staticmethod
def __new__(cls, elem):
out = torch.Tensor._make_wrapper_subclass(
cls,
elem.size(),
dtype=elem.dtype,
layout=elem.layout,
device=elem.device,
requires_grad=elem.requires_grad,
strides=elem.stride(),
storage_offset=elem.storage_offset(),
)
out.elem = elem
return out
def __tensor_flatten__(self):
return ["elem"], None
@staticmethod
def __tensor_unflatten__(inner_tensors, metadata, outer_size, outer_stride):
return _CowStateWrapperTensor(inner_tensors["elem"])
@classmethod
def __torch_dispatch__(cls, func, types, args=(), kwargs=None):
raise AssertionError("unexpected dispatch")
@skipIfTorchDynamo("Registry tests don't need dynamo compilation")
class TestRegistry(TestCase):
"""Tests for the torch._native.registry module."""
def setUp(self):
"""Clean up registry state before each test."""
super().setUp()
self.registry = registry_module
# Store original state for restoration
self._original_libs = dict(self.registry._libs)
self._original_graphs = dict(self.registry._graphs)
self._original_dsl_name_to_lib_graph = {
k: list(v) for k, v in self.registry._dsl_name_to_lib_graph.items()
}
self._original_dispatch_key_to_lib_graph = {
k: list(v) for k, v in self.registry._dispatch_key_to_lib_graph.items()
}
self._original_op_symbol_to_lib_graph = {
k: list(v) for k, v in self.registry._op_symbol_to_lib_graph.items()
}
# Store original filter state
self._original_filter_state = (
set(self.registry._filter_state._dsl_names),
set(self.registry._filter_state._op_symbols),
set(self.registry._filter_state._dispatch_keys),
)
# Clear global state
self.registry._libs.clear()
self.registry._graphs.clear()
self.registry._dsl_name_to_lib_graph.clear()
self.registry._dispatch_key_to_lib_graph.clear()
self.registry._op_symbol_to_lib_graph.clear()
# Clear filter state to ensure clean start
self.registry._filter_state._dsl_names.clear()
self.registry._filter_state._op_symbols.clear()
self.registry._filter_state._dispatch_keys.clear()
def tearDown(self):
"""Restore original registry state after each test."""
if hasattr(self, "registry"):
# Restore original state
self.registry._libs.clear()
self.registry._libs.update(self._original_libs)
self.registry._graphs.clear()
self.registry._graphs.update(self._original_graphs)
# Properly restore mapping dictionaries with new list instances
self.registry._dsl_name_to_lib_graph.clear()
for k, v in self._original_dsl_name_to_lib_graph.items():
self.registry._dsl_name_to_lib_graph[k] = list(v)
self.registry._dispatch_key_to_lib_graph.clear()
for k, v in self._original_dispatch_key_to_lib_graph.items():
self.registry._dispatch_key_to_lib_graph[k] = list(v)
self.registry._op_symbol_to_lib_graph.clear()
for k, v in self._original_op_symbol_to_lib_graph.items():
self.registry._op_symbol_to_lib_graph[k] = list(v)
# Restore filter state
self.registry._filter_state._dsl_names.clear()
self.registry._filter_state._op_symbols.clear()
self.registry._filter_state._dispatch_keys.clear()
self.registry._filter_state._dsl_names.update(
self._original_filter_state[0]
)
self.registry._filter_state._op_symbols.update(
self._original_filter_state[1]
)
self.registry._filter_state._dispatch_keys.update(
self._original_filter_state[2]
)
# Keep essential existing tests
def test_override_node_dataclass(self):
"""Test _OverrideNode dataclass creation and defaults."""
def cond_fn(x):
return True
def impl_fn(x):
return x
node = self.registry._OverrideNode(
"test_dsl", "aten", "add.Tensor", "CPU", cond_fn, impl_fn, "test_node"
)
self.assertEqual(node.dsl_name, "test_dsl")
self.assertEqual(node.lib_symbol, "aten")
self.assertEqual(node.op_symbol, "add.Tensor")
self.assertEqual(node.dispatch_key, "CPU")
self.assertEqual(node.cond_fn, cond_fn)
self.assertEqual(node.impl_fn, impl_fn)
self.assertIsNone(node.try_initialize_runtime)
self.assertFalse(node.unconditional_override)
self.assertTrue(node.active)
@patch("torch.library.Library")
def test_register_op_override_basic(self, mock_library_cls):
"""Test basic register_op_override functionality."""
def cond_fn(x):
return True
def impl_fn(x):
return x
mock_lib = MagicMock()
mock_library_cls.return_value = mock_lib
self.registry.register_op_override(
"test_backend", "aten", "add.Tensor", "CPU", cond_fn, impl_fn
)
key = ("aten", "add.Tensor", "CPU")
self.assertEqual(len(self.registry._graphs[key]), 1)
node = self.registry._graphs[key][0]
self.assertEqual(node.dsl_name, "test_backend")
self.assertEqual(node.cond_fn, cond_fn)
self.assertEqual(node.impl_fn, impl_fn)
@patch("torch.library.Library")
def test_deregister_op_overrides_basic(self, mock_library_cls):
"""Test basic deregister_op_overrides functionality."""
def cond_fn(x):
return True
def impl_fn(x):
return x
mock_lib = MagicMock()
mock_library_cls.return_value = mock_lib
# Register first
self.registry.register_op_override(
"test_backend", "aten", "mul.Tensor", "CPU", cond_fn, impl_fn
)
key = ("aten", "mul.Tensor", "CPU")
self.assertTrue(self.registry._graphs[key][0].active)
# Then deregister
self.registry.deregister_op_overrides(disable_dsl_names="test_backend")
self.assertFalse(self.registry._graphs[key][0].active)
# NEW FUNCTIONALITY TESTS - ONLY THE ESSENTIAL ONES
def test_reorder_graphs_from_user_function_basic(self):
"""Test basic graph reordering functionality."""
# Set up test data
key = ("aten", "test_reorder.Tensor", "CPU")
def cond_fn(x):
return True
def impl_fn(x):
return x
# Create nodes in specific order
nodes = [
self.registry._OverrideNode(
"dsl_c",
"aten",
"test_reorder.Tensor",
"CPU",
cond_fn,
impl_fn,
"node_c",
),
self.registry._OverrideNode(
"dsl_a",
"aten",
"test_reorder.Tensor",
"CPU",
cond_fn,
impl_fn,
"node_a",
),
self.registry._OverrideNode(
"dsl_b",
"aten",
"test_reorder.Tensor",
"CPU",
cond_fn,
impl_fn,
"node_b",
),
]
self.registry._graphs[key] = nodes
# Define alphabetical ordering function
def alphabetical_order(op_symbol, dispatch_key, graph):
return sorted(graph, key=lambda n: n.dsl_name)
# Apply reordering
self.registry.reorder_graphs_from_user_function(alphabetical_order)
# Verify alphabetical order
reordered_graph = self.registry._graphs[key]
actual_names = [node.dsl_name for node in reordered_graph]
self.assertEqual(actual_names, ["dsl_a", "dsl_b", "dsl_c"])
def test_reorder_graphs_from_user_function_error_handling(self):
"""Test error handling in graph reordering."""
# Set up test data
key = ("aten", "test_error.Tensor", "CPU")
def cond_fn(x):
return True
def impl_fn(x):
return x
node = self.registry._OverrideNode(
"test_dsl",
"aten",
"test_error.Tensor",
"CPU",
cond_fn,
impl_fn,
"test_node",
)
original_graph = [node]
self.registry._graphs[key] = original_graph.copy()
# Define failing ordering function
def failing_order_fn(op_symbol, dispatch_key, graph):
raise ValueError("Test exception")
# Should handle the exception gracefully
with self.assertLogs("torch._native.registry", level="WARNING") as log:
self.registry.reorder_graphs_from_user_function(failing_order_fn)
# Verify warning was logged and original graph preserved
self.assertEqual(len(log.records), 1)
self.assertIn("Graph transformation failed", log.records[0].getMessage())
self.assertEqual(self.registry._graphs[key], original_graph)
def test_get_user_ordering_fn_env_var_not_set(self):
"""Test behavior when environment variable is not set."""
with patch.dict("os.environ", {}, clear=True):
from torch._native import get_user_ordering_fn
get_user_ordering_fn.cache_clear()
result = get_user_ordering_fn()
self.assertIsNone(result)
def test_get_user_ordering_fn_invalid_path(self):
"""Test handling of invalid environment variable paths."""
with patch.dict(
"os.environ",
{"TORCH_PYTHON_NATIVE_USER_GRAPH_ORDER_FN": "nonexistent.module.function"},
):
from torch._native import get_user_ordering_fn
get_user_ordering_fn.cache_clear()
with self.assertRaises(ValueError) as cm:
get_user_ordering_fn()
self.assertIn("Could not resolve", str(cm.exception))
def test_integration_reorder_and_register(self):
"""Integration test: reorder then register functionality."""
def cond_fn(x):
return True
def impl_fn1(x):
return x + 1
def impl_fn2(x):
return x + 2
# Register multiple overrides
self.registry.register_op_override(
"backend_z", "aten", "test.Tensor", "CPU", cond_fn, impl_fn1
)
self.registry.register_op_override(
"backend_a", "aten", "test.Tensor", "CPU", cond_fn, impl_fn2
)
key = ("aten", "test.Tensor", "CPU")
# Verify initial order
initial_names = [node.dsl_name for node in self.registry._graphs[key]]
self.assertEqual(initial_names, ["backend_z", "backend_a"])
# Reorder alphabetically
def alphabetical_order(op_symbol, dispatch_key, graph):
return sorted(graph, key=lambda n: n.dsl_name)
self.registry.reorder_graphs_from_user_function(alphabetical_order)
# Verify reordered
final_names = [node.dsl_name for node in self.registry._graphs[key]]
self.assertEqual(final_names, ["backend_a", "backend_z"])
def test_disallowed_dispatch_key_rejected(self):
"""Overrides installed at Meta / Composite* would loop through the
fake kernel's redispatch — `register_op_override` must reject them.
"""
def cond(*a, **k):
return True
def impl(*a, **k):
return None
for bad_key in (
"Meta",
"CompositeImplicitAutograd",
"CompositeExplicitAutograd",
):
with self.assertRaisesRegex(ValueError, "dispatch_key="):
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", bad_key, cond, impl
)
def test_lib_symbol_off_the_allowlist_rejected(self):
"""Namespace support is opt-in; anything not on the allowlist is
rejected at registration time.
`_native` stands in for "not allowlisted" because it can never be
allowlisted -- it holds the ops carrying the override impls.
"""
self.assertNotIn("_native", self.registry._ALLOWED_LIB_SYMBOLS)
with self.assertRaisesRegex(ValueError, "is not overridable"):
self.registry.register_op_override(
"test_dsl",
"_native",
"some_op",
"CPU",
lambda *a, **k: True,
lambda *a, **k: None,
)
def test_cond_none_without_unconditional_override_rejected(self):
"""cond=None is only valid when unconditional_override=True."""
def impl(*a, **k):
return None
with self.assertRaisesRegex(ValueError, "cond must be provided"):
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", "CPU", None, impl
)
def _restore_override_libs(registry, saved):
"""Put `saved` back, minus any entry a test replaced, and say whether any
was replaced.
Registering over an existing (lib, op, key) destroys the library that was
there (`_install_override`), so a handle in the snapshot can be dead by the
time tearDown runs. Restoring it would leave the registry listing a kernel
the dispatcher no longer has; the caller rebuilds those from the restored
graphs instead, having first destroyed the replacement so the rebuild
captures the op's own kernel as its fallback rather than that replacement.
"""
clobbered = [
key for key, lib in saved.items() if registry._override_libs.get(key) is not lib
]
registry._override_libs.clear()
registry._override_libs.update(
{key: lib for key, lib in saved.items() if key not in clobbered}
)
return bool(clobbered)
@skipIfTorchDynamo("Runtime registry tests exercise the dispatcher directly")
class TestRegistryRuntime(TestCase):
"""End-to-end runtime tests that exercise the real dispatcher.
These tests register overrides on real aten ops and therefore must fully
tear down any dispatcher-visible state in tearDown, otherwise a leaked
router will poison other tests in the same process.
"""
def setUp(self):
super().setUp()
self.registry = registry_module
self._saved = {
"graphs": dict(self.registry._graphs),
"libs": dict(self.registry._libs),
"override_libs": dict(self.registry._override_libs),
# Installation writes a compile router here as well as into the
# dispatcher, so a test that registers leaves one behind unless
# this is restored -- and its op may be gone by then.
"decomp_overrides": dict(self.registry._native_decomp_overrides),
"def_libs": dict(self.registry._def_libs),
"defined_native_ops": set(self.registry._defined_native_ops),
"dsl_map": {
k: list(v) for k, v in self.registry._dsl_name_to_lib_graph.items()
},
"op_map": {
k: list(v) for k, v in self.registry._op_symbol_to_lib_graph.items()
},
"dk_map": {
k: list(v) for k, v in self.registry._dispatch_key_to_lib_graph.items()
},
"node_id_counter": self.registry._node_id_counter,
}
self.registry._graphs.clear()
self.registry._dsl_name_to_lib_graph.clear()
self.registry._op_symbol_to_lib_graph.clear()
self.registry._dispatch_key_to_lib_graph.clear()
def tearDown(self):
# Destroy only what this test installed -- anything that is not the
# handle the snapshot holds for its key, which covers both a new key
# and an override installed over an existing one. On a DSL-equipped
# machine `import torch` leaves production overrides live in
# `_override_libs`; destroying those would strip the dispatcher of
# kernels the registry still lists as installed, for the rest of the
# process.
saved_override_libs = self._saved["override_libs"]
for key, lib in list(self.registry._override_libs.items()):
if saved_override_libs.get(key) is not lib:
lib._destroy()
rebuild = _restore_override_libs(self.registry, saved_override_libs)
# _native namespace DEF libraries and the ops defined on them persist
# for the lifetime of the process (torch.library has no "undefine"),
# so we deliberately do not destroy them — later tests will just
# hit the `name in _defined_native_ops` short-circuit.
# We also leave the fake kernels registered for the same reason.
# Restore the rest.
self.registry._graphs.clear()
self.registry._graphs.update(self._saved["graphs"])
self.registry._libs.clear()
self.registry._libs.update(self._saved["libs"])
self.registry._dsl_name_to_lib_graph.clear()
for k, v in self._saved["dsl_map"].items():
self.registry._dsl_name_to_lib_graph[k] = list(v)
self.registry._op_symbol_to_lib_graph.clear()
for k, v in self._saved["op_map"].items():
self.registry._op_symbol_to_lib_graph[k] = list(v)
self.registry._dispatch_key_to_lib_graph.clear()
for k, v in self._saved["dk_map"].items():
self.registry._dispatch_key_to_lib_graph[k] = list(v)
self.registry._native_decomp_overrides.clear()
self.registry._native_decomp_overrides.update(self._saved["decomp_overrides"])
if rebuild:
self.registry._register_all_overrides()
def _install(self, op_symbol, dispatch_key, lib_symbol="aten"):
"""Build the graph then push it through the real registration path."""
self.registry._register_overrides_from_graph(
lib_symbol,
op_symbol,
dispatch_key,
self.registry._graphs[(lib_symbol, op_symbol, dispatch_key)],
)
def test_teardown_restores_the_fallback_of_a_replaced_override(self):
"""An override installed over one that is already there must not
outlive teardown as the rebuilt override's fallback.
`_install_override` destroys the library it replaces, so the snapshot
holds a dead handle and teardown has to rebuild that override from the
saved graph. The rebuild captures whatever kernel is live at that
moment, so leaving the replacement registered makes it the fallback,
and every later call in the process that the restored override declines
lands in a test's impl. Both classes' teardowns follow the same rule:
destroy what is not the snapshot's own handle for the key.
"""
# `torch.equal`, not `assertEqual`: the latter compares through
# `torch.isclose`, which multiplies by rtol and re-enters the very
# override under test with a float argument the captured Tensor-overload
# fallback cannot take.
def mul():
return torch.ops.aten.mul.Tensor(torch.tensor([2.0]), torch.tensor([4.0]))
key = ("aten", "mul.Tensor", "CPU")
try:
# Stands in for an override that was live before setUp. It declines
# every call, so what the op returns is entirely its fallback.
self.registry.register_op_override(
"test_dsl",
"aten",
"mul.Tensor",
"CPU",
lambda *a, **k: False,
lambda a, b: torch.full_like(a, 11.0),
)
self._install("mul.Tensor", "CPU")
self._saved["override_libs"][key] = self.registry._override_libs[key]
self._saved["graphs"][key] = list(self.registry._graphs[key])
self.assertTrue(torch.equal(mul(), torch.tensor([8.0])))
# The test's own override replaces it at the same key.
self.registry.register_op_override(
"test_dsl",
"aten",
"mul.Tensor",
"CPU",
lambda *a, **k: True,
lambda a, b: torch.full_like(a, 99.0),
)
self._install("mul.Tensor", "CPU")
self.assertTrue(torch.equal(mul(), torch.tensor([99.0])))
# What tearDown does, in the order it does it.
saved_override_libs = self._saved["override_libs"]
for k, lib in list(self.registry._override_libs.items()):
if saved_override_libs.get(k) is not lib:
lib._destroy()
rebuild = _restore_override_libs(self.registry, saved_override_libs)
self.registry._graphs.clear()
self.registry._graphs.update(self._saved["graphs"])
if rebuild:
self.registry._register_all_overrides()
self.assertTrue(torch.equal(mul(), torch.tensor([8.0])))
finally:
# The stand-in was added to the snapshot, so drop it again or the
# real tearDown would preserve it for the rest of the process.
self._saved["override_libs"].pop(key, None)
self._saved["graphs"].pop(key, None)
def test_cond_false_falls_through_to_native(self):
"""cond=False must transparently invoke the captured native kernel."""
sentinel_called = [False]
def cond(*a, **k):
return False
def impl(a, b):
sentinel_called[0] = True
return torch.zeros_like(a)
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", "CPU", cond, impl
)
self._install("mul.Tensor", "CPU")
a = torch.tensor([2.0, 3.0])
b = torch.tensor([4.0, 5.0])
# Call the overload directly; `a * b` goes through overload
# resolution which can dispatch to mul.Scalar for mixed args.
out = torch.ops.aten.mul.Tensor(a, b)
self.assertTrue(torch.equal(out, torch.tensor([8.0, 15.0])))
self.assertFalse(sentinel_called[0])
def test_runtime_initialization_falls_through(self):
impl = MagicMock(side_effect=AssertionError("implementation was called"))
self.registry.register_op_override(
"test_dsl",
"aten",
"mul.Tensor",
"CPU",
lambda a, b: True,
impl,
try_initialize_runtime=lambda: False,
)
self._install("mul.Tensor", "CPU")
self.assertEqual(
torch.ops.aten.mul.Tensor(torch.tensor([2.0]), torch.tensor([4.0])).item(),
8.0,
)
impl.assert_not_called()
def test_compile_session_flag_falls_through_without_recursion(self):
"""The eager router must not redispatch to its own aten override when
compile-session state is set but Dynamo is not actively tracing it.
"""
def cond(*a, **k):
return False
def impl(a, b):
raise AssertionError("impl should not be called when cond=False")
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", "CPU", cond, impl
)
self._install("mul.Tensor", "CPU")
a = torch.tensor([2.0, 3.0])
b = torch.tensor([4.0, 5.0])
with torch.compiler._compile_session_context():
out = torch.ops.aten.mul.Tensor(a, b)
self.assertTrue(torch.equal(out, torch.tensor([8.0, 15.0])))
def test_folded_dynamo_flag_on_real_tensors_does_not_recurse(self):
"""A residual eager frame carrying Dynamo's folded
is_dynamo_compiling()==True must NOT divert to the aten overload:
that call re-enters the dispatcher from the top, back into this
router (RecursionError). Real tensors always take eager dispatch.
"""
cond_calls = [0]
def cond(*a, **k):
# Reached only from the re-entrant call the shortcut triggers, so
# the guard must be held here: this also fails if the shortcut is
# removed and cond is consulted directly.
self.assertTrue(getattr(self.registry._router_active, "on", False))
cond_calls[0] += 1
return False
def impl(a, b):
raise AssertionError("impl should not be called when cond=False")
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", "CPU", cond, impl
)
self._install("mul.Tensor", "CPU")
a = torch.tensor([2.0, 3.0])
b = torch.tensor([4.0, 5.0])
# Dynamo folds the call to a True constant while tracing; a frame
# carrying that constant can still run eagerly. patch() reproduces
# that state without needing a compiled residual.
with patch.object(torch.compiler, "is_dynamo_compiling", return_value=True):
out = torch.ops.aten.mul.Tensor(a, b)
self.assertTrue(torch.equal(out, torch.tensor([8.0, 15.0])))
# Eager dispatch ran (cond consulted) exactly once: no runaway re-entry.
self.assertEqual(cond_calls[0], 1)
def test_dynamo_shortcut_still_fires_for_outer_call(self):
"""The re-entrancy guard must not disable the shortcut itself: an
outer call with the folded flag still diverts to aten (that is what
keeps trace-time graph breaks away)."""
cond_calls = [0]
def cond(*a, **k):
cond_calls[0] += 1
return False
def impl(a, b):
raise AssertionError("impl should not be called when cond=False")
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", "CPU", cond, impl
)
self._install("mul.Tensor", "CPU")
a = torch.tensor([2.0, 3.0])
b = torch.tensor([4.0, 5.0])
with patch.object(torch.compiler, "is_dynamo_compiling", return_value=True):
out = torch.ops.aten.mul.Tensor(a, b)
self.assertTrue(torch.equal(out, torch.tensor([8.0, 15.0])))
# Outer call diverted to aten; the re-entrant call it triggers takes
# eager dispatch, consulting cond exactly once (and not recursing).
self.assertEqual(cond_calls[0], 1)
self.assertFalse(getattr(self.registry._router_active, "on", False))
def test_dynamo_shortcut_preserves_cow_fallback(self):
"""COW inputs must keep using the eager fallback path under Dynamo."""
sentinel_called = [False]
def cond(*a, **k):
sentinel_called[0] = True
return False
def impl(a, b):
raise AssertionError("impl should not be called when cond=False")
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", "CPU", cond, impl
)
self._install("mul.Tensor", "CPU")
@torch.compile(backend="eager")
def fn(a, b):
with torch.device("cpu"):
return torch.mul(a, b)
a = torch._lazy_clone(torch.tensor([2.0, 3.0]))
b = torch._lazy_clone(torch.tensor([4.0, 5.0]))
self.assertTrue(torch._C._is_cow_tensor(a))
self.assertTrue(torch._C._is_cow_tensor(b))
out = fn(a, b)
self.assertTrue(torch.equal(out, torch.tensor([8.0, 15.0])))
self.assertTrue(sentinel_called[0])
self.assertTrue(torch._C._is_cow_tensor(a))
self.assertTrue(torch._C._is_cow_tensor(b))
def test_dynamo_guards_on_cow_state(self):
@torch.compile(backend="eager")
def fn(a):
return torch._C._is_cow_tensor(a)
self.assertFalse(fn(torch.tensor([2.0, 3.0])))
self.assertTrue(fn(torch._lazy_clone(torch.tensor([2.0, 3.0]))))
def test_dynamo_graph_breaks_on_sourceless_cow_state(self):
@torch.compile(backend="eager")
def fn(a):
return torch._C._is_cow_tensor(a.view_as(a))
self.assertFalse(fn(torch.tensor([2.0, 3.0])))
self.assertTrue(fn(torch._lazy_clone(torch.tensor([2.0, 3.0]))))
def test_dynamo_graph_breaks_on_mutated_cow_state(self):
@torch.compile(backend="eager")
def fn(a):
a.add_(1)
return torch._C._is_cow_tensor(a)
self.assertFalse(fn(torch._lazy_clone(torch.tensor([2.0, 3.0]))))
def test_dynamo_graph_breaks_after_lazy_clone_changes_cow_state(self):
@torch.compile(backend="inductor")
def fn(a):
clone = a._lazy_clone()
return torch._C._is_cow_tensor(a), torch._C._is_cow_tensor(clone)
self.assertEqual(
fn(torch.tensor([2.0, 3.0])),
(True, True),
)
def test_dynamo_graph_breaks_after_lazy_clone_function_changes_cow_state(self):
fn_id = id(torch._lazy_clone)
prior_trace_rule_state = (
fn_id in trace_rules._allowed_callable_ids,
fn_id in trace_rules._disallowed_callable_ids,
fn_id in trace_rules._nonstrict_trace_callable_ids,
)
# torch._lazy_clone normally stays out of the FX graph. Force it into
# the graph here to cover the defensive call_function scan.
if fn_id in trace_rules._disallowed_callable_ids:
trace_rules._disallowed_callable_ids.remove(fn_id)
trace_rules._allowed_callable_ids.add(fn_id)
try:
torch._dynamo.reset()
@torch.compile(backend="eager")
def fn(a):
clone = torch._lazy_clone(a)
return torch._C._is_cow_tensor(a), torch._C._is_cow_tensor(clone)
self.assertEqual(
fn(torch.tensor([2.0, 3.0])),
(True, True),
)
finally:
if fn_id in trace_rules._allowed_callable_ids:
trace_rules._allowed_callable_ids.remove(fn_id)
if fn_id in trace_rules._disallowed_callable_ids:
trace_rules._disallowed_callable_ids.remove(fn_id)
if fn_id in trace_rules._nonstrict_trace_callable_ids:
trace_rules._nonstrict_trace_callable_ids.remove(fn_id)
if prior_trace_rule_state[0]:
trace_rules._allowed_callable_ids.add(fn_id)
if prior_trace_rule_state[1]:
trace_rules._disallowed_callable_ids.add(fn_id)
if prior_trace_rule_state[2]:
trace_rules._nonstrict_trace_callable_ids.add(fn_id)
torch._dynamo.reset()
def test_dynamo_graph_breaks_after_lazy_clone_view_changes_cow_state(self):
@torch.compile(backend="eager")
def fn(a):
view = a.view(-1)
clone = view._lazy_clone()
return (
torch._C._is_cow_tensor(a),
torch._C._is_cow_tensor(view),
torch._C._is_cow_tensor(clone),
)
self.assertEqual(
fn(torch.tensor([2.0, 3.0])),
(True, True, True),
)
def test_cow_guard_misses_on_fake_tensor(self):
from torch._dynamo.guards import _cow_tensor_matches
with FakeTensorMode():
fake = torch.empty(2)
self.assertFalse(_cow_tensor_matches(fake, False))
self.assertFalse(_cow_tensor_matches(fake, True))
def test_is_cow_tensor_rejects_python_tensor_subclasses(self):
x = _CowStateWrapperTensor(torch.tensor([2.0, 3.0]))
with self.assertRaisesRegex(
RuntimeError, "_is_cow_tensor is not defined for Python tensor subclasses"
):
torch._C._is_cow_tensor(x)
def test_dynamo_graph_breaks_on_subclass_cow_state(self):
@torch.compile(backend="eager", fullgraph=True)
def fn(a):
return torch._C._is_cow_tensor(a)
x = _CowStateWrapperTensor(torch.tensor([2.0, 3.0]))
with self.assertRaisesRegex(
Exception, "COW tensor check on Python tensor subclass"
):
fn(x)
def test_dynamo_allows_previously_mutated_cow_state(self):
@torch.compile(backend="eager", fullgraph=True)
def fn(a):
return torch._C._is_cow_tensor(a)
x = torch.tensor([2.0, 3.0])
x.add_(1)
self.assertFalse(fn(x))
def test_strict_export_rejects_cow_state(self):
class Mod(torch.nn.Module):
def forward(self, a):
if torch._C._is_cow_tensor(a):
return a + 1
return a + 2
with self.assertRaisesRegex(Exception, "COW tensor check during export"):
torch.export.export(
Mod(),
(torch._lazy_clone(torch.tensor([2.0, 3.0])),),
strict=True,
)
def test_non_strict_export_rejects_cow_state(self):
class Mod(torch.nn.Module):
def forward(self, a):
if torch._C._is_cow_tensor(a):
return a + 1
return a + 2
with self.assertRaisesRegex(
Exception, "_is_cow_tensor is not defined for Python tensor subclasses"
):
torch.export.export(
Mod(),
(torch._lazy_clone(torch.tensor([2.0, 3.0])),),
strict=False,
)
def test_cond_true_routes_to_impl(self):
"""cond=True must route the call to the registered impl."""
def cond(*a, **k):
return True
def impl(a, b):
return torch.full_like(a, 42.0)
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", "CPU", cond, impl
)
self._install("mul.Tensor", "CPU")
a = torch.tensor([2.0, 3.0])
b = torch.tensor([4.0, 5.0])
out = torch.ops.aten.mul.Tensor(a, b)
self.assertTrue(torch.equal(out, torch.tensor([42.0, 42.0])))
def test_no_recursion_in_aten_backward_formula(self):
"""Backward of bmm calls bmm again; the fallback must bypass the router.
If the router weren't bypassed, aten's bmm backward formula would
re-enter it on every autograd step, either recursing forever or
silently calling the override impl in the grad path.
"""
def cond(*a, **k):
# False so every call hits the fallback — this isolates the
# "native kernel called via fallback must not re-enter the router"
# property.
return False
def impl(a, b):
raise AssertionError("impl should not be called when cond=False")
self.registry.register_op_override("test_dsl", "aten", "bmm", "CPU", cond, impl)
self._install("bmm", "CPU")
a = torch.randn(2, 3, 4, requires_grad=True)
b = torch.randn(2, 4, 5, requires_grad=True)
expected = a.detach() @ b.detach()
out = torch.bmm(a, b)
self.assertEqual(out, expected)
out.sum().backward()
self.assertEqual(a.grad.shape, a.shape)
self.assertEqual(b.grad.shape, b.shape)
self.assertTrue((a.grad != 0).any())
self.assertTrue((b.grad != 0).any())
def test_deregister_reenable_roundtrip(self):
"""deregister must tear down the aten router; reenable must reinstall."""
def cond(*a, **k):
return True
def impl(a, b):
return torch.full_like(a, 7.0)
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", "CPU", cond, impl
)
self._install("mul.Tensor", "CPU")
a = torch.tensor([2.0, 3.0])
b = torch.tensor([4.0, 5.0])
mul = torch.ops.aten.mul.Tensor
# Override active.
self.assertTrue(torch.equal(mul(a, b), torch.tensor([7.0, 7.0])))
self.assertIn(("aten", "mul.Tensor", "CPU"), self.registry._override_libs)
# Deregister → native kernel returns.
self.registry.deregister_op_overrides(disable_dsl_names="test_dsl")
self.assertTrue(torch.equal(mul(a, b), torch.tensor([8.0, 15.0])))
self.assertNotIn(("aten", "mul.Tensor", "CPU"), self.registry._override_libs)
# Reenable → override fires again.
self.registry.reenable_op_overrides(enable_dsl_names="test_dsl")
self.assertTrue(torch.equal(mul(a, b), torch.tensor([7.0, 7.0])))
self.assertIn(("aten", "mul.Tensor", "CPU"), self.registry._override_libs)
def test_empty_graph_tears_down_router(self):
"""An empty graph passed to _cleanup_and_reregister_graph must still
tear down the previously-installed aten router.
Regression test: before the fix, the `if graph:` guard skipped
_register_overrides_from_graph entirely, leaving the aten override
live with a stale closure.
"""
def cond(*a, **k):
return True
def impl(a, b):
return torch.full_like(a, 99.0)
self.registry.register_op_override(
"test_dsl", "aten", "mul.Tensor", "CPU", cond, impl
)
self._install("mul.Tensor", "CPU")
a = torch.tensor([2.0, 3.0])
b = torch.tensor([4.0, 5.0])
mul = torch.ops.aten.mul.Tensor
self.assertTrue(torch.equal(mul(a, b), torch.tensor([99.0, 99.0])))
self.assertIn(("aten", "mul.Tensor", "CPU"), self.registry._override_libs)
# Simulate a filter-out-everything transformation.
self.registry._graphs[("aten", "mul.Tensor", "CPU")] = []
self.registry._cleanup_and_reregister_graph(
"aten",
"mul.Tensor",
"CPU",
self.registry._graphs[("aten", "mul.Tensor", "CPU")],
)
self.assertNotIn(("aten", "mul.Tensor", "CPU"), self.registry._override_libs)
self.assertTrue(torch.equal(mul(a, b), torch.tensor([8.0, 15.0])))
def test_fake_tensor_shape_inference(self):
"""FakeTensor shape-infers through `_native::<id>` via the registered
fake kernel, without running the implementation or runtime initialization.
"""
def cond(*a, **k):
return True
impl = MagicMock(side_effect=AssertionError("implementation ran"))
try_initialize_runtime = MagicMock(
side_effect=AssertionError("runtime initialization ran")
)
self.registry.register_op_override(
"test_dsl",
"aten",
"mul.Tensor",
"CPU",
cond,
impl,
try_initialize_runtime=try_initialize_runtime,
)
self._install("mul.Tensor", "CPU")
with FakeTensorMode():
a = torch.empty(3, 4, dtype=torch.float32)
b = torch.empty(3, 4, dtype=torch.float32)
out = torch.ops.aten.mul.Tensor(a, b)
self.assertEqual(out.shape, torch.Size([3, 4]))
self.assertEqual(out.dtype, torch.float32)
def test_unconditional_override_cond_none(self):
"""`cond=None` + `unconditional_override=True` must substitute a
trivially-true predicate so the impl fires on every call.
"""
call_count = [0]
def impl(a, b):
call_count[0] += 1
return torch.full_like(a, 5.0)
self.registry.register_op_override(
"test_dsl",
"aten",
"mul.Tensor",
"CPU",
None,
impl,
unconditional_override=True,
)
self._install("mul.Tensor", "CPU")
a = torch.tensor([2.0, 3.0])
b = torch.tensor([4.0, 5.0])
self.assertTrue(
torch.equal(torch.ops.aten.mul.Tensor(a, b), torch.tensor([5.0, 5.0]))
)
self.assertTrue(
torch.equal(
torch.ops.aten.mul.Tensor(torch.tensor([0.0]), torch.tensor([0.0])),
torch.tensor([5.0]),
)
)
self.assertEqual(call_count[0], 2)
def test_unconditional_override_rejects_runtime_initialization(self):
with self.assertRaisesRegex(
ValueError, "try_initialize_runtime cannot be provided"
):
self.registry.register_op_override(
"test_dsl",
"aten",
"mul.Tensor",
"CPU",
None,
lambda a, b: a,
try_initialize_runtime=lambda: True,
unconditional_override=True,
)
# torch.equal, not assertEqual, below: assertEqual computes its tolerances with
# tensor*float, which this UNCONDITIONAL override on mul.Tensor also catches, so
# the comparison helper would run through the override under test.
def _register_unconditional_mul(self):
def impl(a, b):
return torch.full_like(a, 5.0)
self.registry.register_op_override(
"test_dsl",
"aten",
"mul.Tensor",
"CPU",
None,
impl,
unconditional_override=True,
)
self._install("mul.Tensor", "CPU")
return torch.tensor([2.0, 3.0]), torch.tensor([4.0, 5.0])
def test_unconditional_override_survives_disable(self):
"""An unconditional override IS the implementation, so the user-facing disable
must leave it installed: masking it changes what the op computes."""
a, b = self._register_unconditional_mul()
overridden = torch.tensor([5.0, 5.0])
self.assertTrue(torch.equal(torch.ops.aten.mul.Tensor(a, b), overridden))
try:
self.registry.deregister_op_overrides(disable_dsl_names="test_dsl")
self.assertTrue(torch.equal(torch.ops.aten.mul.Tensor(a, b), overridden))
finally:
self.registry.reenable_op_overrides(enable_dsl_names="test_dsl")
def test_private_hatch_masks_unconditional_override(self):
"""The reference-computation hatch is the one thing that may mask it. It only
lifts the exemption, so the ordinary disable must also be in effect."""
a, b = self._register_unconditional_mul()
overridden = torch.tensor([5.0, 5.0])
stock = torch.tensor([8.0, 15.0])
try:
self.registry.deregister_op_overrides(disable_dsl_names="test_dsl")
with torch._native._unconditional_masked():
self.assertTrue(torch.equal(torch.ops.aten.mul.Tensor(a, b), stock))
# The hatch is scoped, so exiting restores it.
self.assertTrue(torch.equal(torch.ops.aten.mul.Tensor(a, b), overridden))
finally:
self.registry.reenable_op_overrides(enable_dsl_names="test_dsl")
@skipIfTorchDynamo("Runtime registry tests exercise the dispatcher directly")
class TestRegistryNonAtenNamespace(TestCase):
"""Overrides on namespaces other than `aten`.
Mirrors how a Python-defined op looks to the registry: the op's own
implementation sits at CompositeExplicitAutograd (what
`torch.library.custom_op` produces), so the override is installed at the
backend key and the captured fallback resolves to the composite kernel.
These ops carry no Autograd kernel above the router, so the autograd
layer -- the shape a real `torch.library.custom_op` takes, and the
motivation for the widened keys -- is not covered here. That coverage
lives with the first torch_nn override and its own test.
"""
NS = "_native_registry_test"
NS2 = "_native_registry_test2"
MISSING_NS = "_native_registry_missing_ns"
def setUp(self):
super().setUp()
self.registry = registry_module
self._saved_graphs = dict(self.registry._graphs)
self._saved_override_libs = dict(self.registry._override_libs)
# Installing an override also records a compile router keyed on the
# overload (`_native_decomp_overrides`), which `native_decomp_table()`
# merges. Without this, a router for an op destroyed with the scoped
# library below outlives the test.
self._saved_decomp_overrides = dict(self.registry._native_decomp_overrides)
self._saved_maps = {
name: {k: list(v) for k, v in getattr(self.registry, name).items()}
for name in (
"_dsl_name_to_lib_graph",
"_op_symbol_to_lib_graph",
"_dispatch_key_to_lib_graph",
)
}
self.registry._graphs.clear()
# These throwaway namespaces are not on the production allowlist.
self._stack = contextlib.ExitStack()
self._stack.enter_context(
patch.object(
self.registry,
"_ALLOWED_LIB_SYMBOLS",
self.registry._ALLOWED_LIB_SYMBOLS
| {self.NS, self.NS2, self.MISSING_NS},
)
)
# A throwaway namespace with one op whose only kernel is composite,
# matching the shape of a `torch.library.custom_op` definition.
self.lib = self._stack.enter_context(
torch.library._scoped_library(self.NS, "DEF")
)
self.lib.define("twice(Tensor self) -> Tensor")
self.lib.impl("twice", lambda x: x * 2, "CompositeExplicitAutograd")
self.op = getattr(torch.ops, self.NS).twice.default
# A second namespace defining the SAME op symbol, so the tests below
# exercise the per-op-symbol maps the widened keys share. Its
# fallback triples rather than doubles, to tell the two apart.
self.lib2 = self._stack.enter_context(
torch.library._scoped_library(self.NS2, "DEF")
)
self.lib2.define("twice(Tensor self) -> Tensor")
self.lib2.impl("twice", lambda x: x * 3, "CompositeExplicitAutograd")
self.op2 = getattr(torch.ops, self.NS2).twice.default
self._saved_filter_state = (
set(self.registry._filter_state._dsl_names),
set(self.registry._filter_state._op_symbols),
set(self.registry._filter_state._dispatch_keys),
)
def tearDown(self):
# Only what this test installed; production overrides in the snapshot
# stay live (see TestRegistryRuntime.tearDown).
for key, lib in list(self.registry._override_libs.items()):
if self._saved_override_libs.get(key) is not lib:
lib._destroy()
rebuild = _restore_override_libs(self.registry, self._saved_override_libs)
self._stack.close()
dsl_names, op_symbols, dispatch_keys = self._saved_filter_state
for attr, saved in (
("_dsl_names", dsl_names),
("_op_symbols", op_symbols),
("_dispatch_keys", dispatch_keys),
):
target = getattr(self.registry._filter_state, attr)
target.clear()
target.update(saved)
self.registry._graphs.clear()
self.registry._graphs.update(self._saved_graphs)
self.registry._native_decomp_overrides.clear()
self.registry._native_decomp_overrides.update(self._saved_decomp_overrides)
if rebuild:
self.registry._register_all_overrides()
for name, saved in self._saved_maps.items():
target = getattr(self.registry, name)
target.clear()
for k, v in saved.items():
target[k] = list(v)
super().tearDown()
def _install(self, lib_symbol, op_symbol="twice", dispatch_key="CPU"):
key = (lib_symbol, op_symbol, dispatch_key)
self.registry._register_overrides_from_graph(
lib_symbol, op_symbol, dispatch_key, self.registry._graphs[key]
)
def _register(self, cond, impl, lib_symbol=None):
self.registry.register_op_override(
"test_dsl", lib_symbol or self.NS, "twice", "CPU", cond, impl
)
def test_graph_key_and_node_carry_the_namespace(self):
self._register(lambda *a, **k: True, lambda x: x)
key = (self.NS, "twice", "CPU")
self.assertIn(key, self.registry._graphs)
self.assertEqual(self.registry._graphs[key][0].lib_symbol, self.NS)
def test_cond_true_routes_to_impl(self):
self._register(lambda *a, **k: True, lambda x: torch.full_like(x, 99.0))
self._install(self.NS)
self.assertEqual(self.op(torch.tensor([1.0, 2.0])), torch.tensor([99.0, 99.0]))
def test_cond_false_falls_back_to_composite_kernel(self):
def impl(x):
raise AssertionError("impl must not run when cond is False")
self._register(lambda *a, **k: False, impl)
self._install(self.NS)
self.assertEqual(self.op(torch.tensor([1.0, 2.0])), torch.tensor([2.0, 4.0]))
def test_teardown_restores_the_original_kernel(self):
self._register(lambda *a, **k: True, lambda x: torch.full_like(x, 99.0))
self._install(self.NS)
self.assertEqual(self.op(torch.tensor([1.0])), torch.tensor([99.0]))
self.registry._graphs[(self.NS, "twice", "CPU")] = []
self._install(self.NS)
self.assertNotIn((self.NS, "twice", "CPU"), self.registry._override_libs)
self.assertEqual(self.op(torch.tensor([1.0])), torch.tensor([2.0]))
def test_teardown_leaves_no_decomposition_router(self):
"""Installation records a compile router in `_native_decomp_overrides`
as well as in the dispatcher, and `native_decomp_table()` merges that
dict. The scoped op it is keyed on dies with this class's library, so a
router left behind would outlive its op and shadow later work through
the decomposition table.
Written as a check of the state this class's own tearDown restores,
because the leak is only observable after tearDown has run -- the
instance is gone by then, so the assertion lives in the class that
installs the router rather than in a later test that happens to trip
over it.
"""
before = dict(self.registry._native_decomp_overrides)
self._register(lambda *a, **k: True, lambda x: torch.full_like(x, 99.0))
self._install(self.NS)
self.assertNotEqual(
dict(self.registry._native_decomp_overrides),
before,
"installing an override should record a decomposition router",
)
# What tearDown does, in the order it does it.
for key, lib in list(self.registry._override_libs.items()):
if self._saved_override_libs.get(key) is not lib:
lib._destroy()
_restore_override_libs(self.registry, self._saved_override_libs)
self.registry._graphs.clear()
self.registry._graphs.update(self._saved_graphs)
self.registry._native_decomp_overrides.clear()
self.registry._native_decomp_overrides.update(self._saved_decomp_overrides)
self.assertEqual(dict(self.registry._native_decomp_overrides), before)
self.assertNotIn(self.op, self.registry._native_decomp_overrides)
def _register_both(self):
self._register(lambda *a, **k: True, lambda x: torch.full_like(x, 99.0))
self._register(
lambda *a, **k: True,
lambda x: torch.full_like(x, 7.0),
lib_symbol=self.NS2,
)
self._install(self.NS)
self._install(self.NS2)
def test_same_op_symbol_in_two_namespaces_is_independent(self):
"""Two namespaces defining the same op symbol land in one
per-op-symbol bucket; each must still route to its own override."""
self._register_both()
x = torch.tensor([1.0, 2.0])
self.assertEqual(self.op(x), torch.full_like(x, 99.0))
self.assertEqual(self.op2(x), torch.full_like(x, 7.0))
bucket = self.registry._op_symbol_to_lib_graph["twice"]
self.assertIn((self.NS, "twice", "CPU"), bucket)
self.assertIn((self.NS2, "twice", "CPU"), bucket)
def test_op_symbol_filters_are_namespace_blind(self):
"""`disable_op_symbols` matches the bare op symbol in every namespace,
and a "ns::op"-qualified string matches nothing. Pinned here because
the widened keys make same-symbol collisions across namespaces
possible for the first time."""
self._register_both()
x = torch.tensor([1.0, 2.0])
self.registry.deregister_op_overrides(disable_op_symbols=[f"{self.NS}::twice"])
self.assertEqual(self.op(x), torch.full_like(x, 99.0))
self.assertEqual(self.op2(x), torch.full_like(x, 7.0))
self.registry.deregister_op_overrides(disable_op_symbols=["twice"])
self.assertEqual(self.op(x), x * 2)
self.assertEqual(self.op2(x), x * 3)
self.registry.reenable_op_overrides(enable_op_symbols=["twice"])
self.assertEqual(self.op(x), torch.full_like(x, 99.0))
self.assertEqual(self.op2(x), torch.full_like(x, 7.0))
def test_get_dsl_operations_reports_one_namespace(self):
"""Reporting is scoped: the throwaway namespace's op appears only when
asked for, and never in the default `aten` view."""
self._register(lambda *a, **k: True, lambda x: x)
self.assertEqual(
self.registry.get_dsl_operations("test_dsl", lib_symbol=self.NS),
["twice"],
)
self.assertNotIn("twice", self.registry.get_dsl_operations("test_dsl"))
def test_undefined_op_raises_on_install(self):
"""A namespace whose op is not in the dispatcher must fail loudly at
install time rather than silently registering nothing.
Loud failure is the deliberate contract: a registration naming an op
that does not resolve breaks `import torch` rather than degrading
quietly. Unlike `aten`, whose ops the build guarantees, a non-aten
op's existence is a runtime property, so drift is caught before it
ships by the drift-guard test that accompanies each such override.
"""
self._register(lambda *a, **k: True, lambda x: x, lib_symbol=self.MISSING_NS)
with self.assertRaisesRegex(AttributeError, "not found; is the namespace"):
self._install(self.MISSING_NS)
if __name__ == "__main__":
run_tests()