diff --git a/cuda_bindings/tests/test_cufile.py b/cuda_bindings/tests/test_cufile.py index 46bd8429a62..7de4ceb2e20 100644 --- a/cuda_bindings/tests/test_cufile.py +++ b/cuda_bindings/tests/test_cufile.py @@ -138,7 +138,7 @@ def ctx(): (err,) = cuda.cuCtxSetCurrent(ctx) assert err == cuda.CUresult.CUDA_SUCCESS - yield + yield ctx cuda.cuDevicePrimaryCtxRelease(device) diff --git a/cuda_bindings/tests/test_examples.py b/cuda_bindings/tests/test_examples.py index 652515830f8..a7a0524a217 100644 --- a/cuda_bindings/tests/test_examples.py +++ b/cuda_bindings/tests/test_examples.py @@ -19,6 +19,7 @@ def test_example(example): env = os.environ.copy() env["CUDA_BINDINGS_SKIP_EXAMPLE"] = "100" + env["MPLBACKEND"] = "Agg" # avoid plt.show() from blocking process = subprocess.run([sys.executable, example], capture_output=True, env=env) # noqa: S603 # returncode is a special value used in the examples to indicate that system requirements are not met. diff --git a/cuda_core/tests/graph/test_graph_definition.py b/cuda_core/tests/graph/test_graph_definition.py index 4b3122c763e..f8f427567a0 100644 --- a/cuda_core/tests/graph/test_graph_definition.py +++ b/cuda_core/tests/graph/test_graph_definition.py @@ -577,20 +577,6 @@ def node_spec(request, init_cuda): # ============================================================================= -@pytest.fixture -def sample_graphdef(init_cuda): - """A sample GraphDefinition for standalone tests.""" - return GraphDefinition() - - -@pytest.fixture -def dot_file(tmp_path): - """Temporary DOT file path, cleaned up after test.""" - path = tmp_path / "graph.dot" - yield path - path.unlink(missing_ok=True) - - # ============================================================================= # Topology tests (parameterized over graph specs) # ============================================================================= @@ -830,14 +816,16 @@ def registered(node): # ============================================================================= -def test_graphdef_handle_valid(sample_graphdef): +def test_graphdef_handle_valid(init_cuda): """GraphDefinition has a valid non-null handle.""" + sample_graphdef = GraphDefinition() assert sample_graphdef.handle is not None assert int(sample_graphdef.handle) != 0 -def test_graphdef_entry_is_virtual(sample_graphdef): +def test_graphdef_entry_is_virtual(init_cuda): """Internal entry node is virtual (no pred/succ, type is None).""" + sample_graphdef = GraphDefinition() entry = sample_graphdef._entry assert isinstance(entry, GraphNode) assert entry.pred == set() @@ -850,8 +838,9 @@ def test_graphdef_entry_is_virtual(sample_graphdef): # ============================================================================= -def test_alloc_zero_size_fails(sample_graphdef): +def test_alloc_zero_size_fails(init_cuda): """Alloc with zero size raises error (CUDA limitation).""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() from cuda.core._utils.cuda_utils import CUDAError @@ -859,8 +848,9 @@ def test_alloc_zero_size_fails(sample_graphdef): sample_graphdef.allocate(0) -def test_free_creates_dependency(sample_graphdef): +def test_free_creates_dependency(init_cuda): """Free node depends on its predecessor.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -868,8 +858,9 @@ def test_free_creates_dependency(sample_graphdef): assert alloc in free.pred -def test_alloc_free_chain(sample_graphdef): +def test_alloc_free_chain(init_cuda): """Alloc and free can be chained.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): a1 = sample_graphdef.allocate(ALLOC_SIZE) @@ -886,8 +877,9 @@ def test_alloc_free_chain(sample_graphdef): # ============================================================================= -def test_alloc_memory_type_invalid(sample_graphdef): +def test_alloc_memory_type_invalid(init_cuda): """Invalid memory type raises ValueError.""" + sample_graphdef = GraphDefinition() with pytest.raises(ValueError, match="'invalid' is not a valid GraphMemoryType. Must be "): sample_graphdef.allocate(ALLOC_SIZE, memory_type="invalid") @@ -899,8 +891,9 @@ def test_alloc_memory_type_invalid(sample_graphdef): pytest.param(lambda d: d, id="Device_object"), ], ) -def test_alloc_device_option(sample_graphdef, device_spec): +def test_alloc_device_option(init_cuda, device_spec): """Device can be specified as int or Device object.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() device = Device() with xfail_on_graph_mempool_oom(device): @@ -923,8 +916,9 @@ def test_alloc_peer_access(mempool_device_x2): @pytest.mark.parametrize("num_branches", [2, 3, 5]) -def test_join_merges_branches(sample_graphdef, num_branches): +def test_join_merges_branches(init_cuda, num_branches): """join() with multiple branches creates correct dependencies.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)] @@ -938,8 +932,9 @@ def test_join_merges_branches(sample_graphdef, num_branches): # ============================================================================= -def test_launch_creates_node(sample_graphdef): +def test_launch_creates_node(init_cuda): """launch() creates a KernelNode.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -947,8 +942,9 @@ def test_launch_creates_node(sample_graphdef): assert isinstance(node, KernelNode) -def test_launch_chain_dependencies(sample_graphdef): +def test_launch_chain_dependencies(init_cuda): """Chained launches create correct dependencies.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -1010,15 +1006,17 @@ def _instantiate_and_upload(graph_definition, kwargs, stream): @pytest.mark.parametrize("inst_kwargs", _INSTANTIATE_ONLY_OPTIONS) -def test_instantiate_empty_graph(sample_graphdef, inst_kwargs): +def test_instantiate_empty_graph(init_cuda, inst_kwargs): """Empty graph can be instantiated.""" + sample_graphdef = GraphDefinition() graph = _instantiate(sample_graphdef, inst_kwargs) assert graph is not None @pytest.mark.parametrize("inst_kwargs", _INSTANTIATE_ONLY_OPTIONS) -def test_instantiate_with_nodes(sample_graphdef, inst_kwargs): +def test_instantiate_with_nodes(init_cuda, inst_kwargs): """Graph with nodes can be instantiated.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) @@ -1028,8 +1026,9 @@ def test_instantiate_with_nodes(sample_graphdef, inst_kwargs): @pytest.mark.skipif(not Device(0).properties.unified_addressing, reason="requires unified addressing") -def test_instantiate_and_execute_kernel_device_launch(sample_graphdef): +def test_instantiate_and_execute_kernel_device_launch(init_cuda): """Kernel-only graph can be instantiated with device_launch flag.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -1045,8 +1044,9 @@ def test_instantiate_and_execute_kernel_device_launch(sample_graphdef): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_kernel(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_kernel(init_cuda, inst_kwargs): """Graph with kernel can be instantiated and executed.""" + sample_graphdef = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") config = LaunchConfig(grid=1, block=1) @@ -1059,8 +1059,9 @@ def test_instantiate_and_execute_kernel(sample_graphdef, inst_kwargs): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs): """Graph with alloc/free can be executed.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -1073,8 +1074,9 @@ def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_memset(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_memset(init_cuda, inst_kwargs): """Graph with alloc/memset/free can be executed.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -1088,8 +1090,9 @@ def test_instantiate_and_execute_memset(sample_graphdef, inst_kwargs): @pytest.mark.parametrize("inst_kwargs", _EXECUTE_OPTIONS) -def test_instantiate_and_execute_memcpy(sample_graphdef, inst_kwargs): +def test_instantiate_and_execute_memcpy(init_cuda, inst_kwargs): """Graph with alloc/memset/memcpy/free can be executed and data is copied.""" + sample_graphdef = GraphDefinition() _skip_if_no_mempool() import ctypes @@ -1113,8 +1116,9 @@ def test_instantiate_and_execute_memcpy(sample_graphdef, inst_kwargs): assert all(b == 0xAB for b in host_buf) -def test_instantiate_and_execute_child_graph(sample_graphdef): +def test_instantiate_and_execute_child_graph(init_cuda): """Graph with embedded child graph can be executed.""" + sample_graphdef = GraphDefinition() child = GraphDefinition() mod = compile_common_kernels() kernel = mod.get_kernel("empty_kernel") @@ -1130,8 +1134,9 @@ def test_instantiate_and_execute_child_graph(sample_graphdef): stream.sync() -def test_instantiate_and_execute_host_callback(sample_graphdef): +def test_instantiate_and_execute_host_callback(init_cuda): """Graph with host callback can be executed and callback is invoked.""" + sample_graphdef = GraphDefinition() results = [] def my_callback(): @@ -1148,8 +1153,9 @@ def my_callback(): assert results == [42] -def test_instantiate_and_execute_host_callback_cfunc(sample_graphdef): +def test_instantiate_and_execute_host_callback_cfunc(init_cuda): """Graph with ctypes function pointer callback can be executed.""" + sample_graphdef = GraphDefinition() import ctypes CALLBACK = ctypes.CFUNCTYPE(None, ctypes.c_void_p) @@ -1170,8 +1176,9 @@ def raw_fn(data): assert called[0] -def test_host_callback_cfunc_with_user_data(sample_graphdef): +def test_host_callback_cfunc_with_user_data(init_cuda): """Host callback with bytes user_data passes data to C function.""" + sample_graphdef = GraphDefinition() import ctypes CALLBACK = ctypes.CFUNCTYPE(None, ctypes.c_void_p) @@ -1192,8 +1199,9 @@ def read_byte(data): assert result[0] == 0xAB -def test_host_callback_user_data_rejected_for_python_callable(sample_graphdef): +def test_host_callback_user_data_rejected_for_python_callable(init_cuda): """user_data is rejected for Python callables.""" + sample_graphdef = GraphDefinition() with pytest.raises(ValueError, match="user_data is only supported"): sample_graphdef.callback(lambda: None, user_data=b"hello") @@ -1216,15 +1224,17 @@ def test_host_callback_user_data_rejected_for_python_callable(sample_graphdef): @pytest.mark.agent_authored(model="cursor-grok-4.5") @pytest.mark.parametrize("callback_type", _INCOMPATIBLE_CTYPES_HOST_CALLBACKS) -def test_host_callback_ctypes_rejects_incompatible_signature(sample_graphdef, callback_type): +def test_host_callback_ctypes_rejects_incompatible_signature(init_cuda, callback_type): """Incompatible ctypes prototypes are rejected before CUDA sees them.""" + sample_graphdef = GraphDefinition() with pytest.raises(TypeError, match="CUhostFn"): sample_graphdef.callback(callback_type(0)) @pytest.mark.agent_authored(model="cursor-grok-4.5") -def test_host_callback_ctypes_update_rejects_incompatible_signature(sample_graphdef): +def test_host_callback_ctypes_update_rejects_incompatible_signature(init_cuda): """HostCallbackNode.update applies the same ctypes ABI check.""" + sample_graphdef = GraphDefinition() good_type = ctypes.CFUNCTYPE(None, ctypes.c_void_p) bad_type = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p) @@ -1239,8 +1249,9 @@ def good(data): @pytest.mark.agent_authored(model="claude-opus-5") @pytest.mark.parametrize("callback_type", _COMPATIBLE_CTYPES_HOST_CALLBACKS) -def test_host_callback_ctypes_accepts_equivalent_prototypes(sample_graphdef, callback_type): +def test_host_callback_ctypes_accepts_equivalent_prototypes(init_cuda, callback_type): """Prototypes that declare CUhostFn are accepted and run.""" + sample_graphdef = GraphDefinition() called = [False] @callback_type @@ -1260,8 +1271,9 @@ def raw_fn(data): @pytest.mark.agent_authored(model="cursor-grok-4.5") @pytest.mark.skipif(sys.platform != "win32", reason="WINFUNCTYPE is Windows-only") -def test_host_callback_ctypes_accepts_winfunctype(sample_graphdef): +def test_host_callback_ctypes_accepts_winfunctype(init_cuda): """On Windows, WINFUNCTYPE matches CUDA_CB (__stdcall) and is accepted.""" + sample_graphdef = GraphDefinition() callback_type = ctypes.WINFUNCTYPE(None, ctypes.c_void_p) called = [False] @@ -1280,8 +1292,9 @@ def raw_fn(data): assert called[0] -def test_instantiate_and_execute_event_record_wait(sample_graphdef): +def test_instantiate_and_execute_event_record_wait(init_cuda): """Graph with event record and wait nodes can be executed.""" + sample_graphdef = GraphDefinition() event = Device().create_event() rec = sample_graphdef.record(event) rec.wait(event) @@ -1303,8 +1316,9 @@ def _skip_unless_cc_90(): pytest.skip("Conditional node execution requires CC >= 9.0 (Hopper)") -def test_instantiate_and_execute_if_then(sample_graphdef): +def test_instantiate_and_execute_if_then(init_cuda): """If-conditional node: body executes only when condition is non-zero.""" + sample_graphdef = GraphDefinition() _skip_unless_cc_90() _skip_if_no_mempool() import ctypes @@ -1336,8 +1350,9 @@ def test_instantiate_and_execute_if_then(sample_graphdef): assert result[0] == 1 -def test_instantiate_and_execute_if_else(sample_graphdef): +def test_instantiate_and_execute_if_else(init_cuda): """If-else node: then or else branch executes based on condition.""" + sample_graphdef = GraphDefinition() _skip_unless_cc_90() _skip_if_no_mempool() import ctypes @@ -1371,8 +1386,9 @@ def test_instantiate_and_execute_if_else(sample_graphdef): assert result[0] == 2 -def test_instantiate_and_execute_switch(sample_graphdef): +def test_instantiate_and_execute_switch(init_cuda): """Switch node: selected branch executes based on condition value.""" + sample_graphdef = GraphDefinition() _skip_unless_cc_90() _skip_if_no_mempool() import ctypes @@ -1405,8 +1421,9 @@ def test_instantiate_and_execute_switch(sample_graphdef): assert result[0] == 1 -def test_conditional_node_type_preserved_by_nodes(sample_graphdef): +def test_conditional_node_type_preserved_by_nodes(init_cuda): """Conditional nodes appear as ConditionalNode base when read back from graph.""" + sample_graphdef = GraphDefinition() condition = try_create_condition(sample_graphdef) if_node = sample_graphdef.if_then(condition) assert isinstance(if_node, IfNode) @@ -1422,8 +1439,10 @@ def test_conditional_node_type_preserved_by_nodes(sample_graphdef): # ============================================================================= -def test_debug_dot_print_creates_file(sample_graphdef, dot_file): +def test_debug_dot_print_creates_file(init_cuda, tmp_path): """debug_dot_print writes a DOT file.""" + sample_graphdef = GraphDefinition() + dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) @@ -1433,8 +1452,10 @@ def test_debug_dot_print_creates_file(sample_graphdef, dot_file): assert "digraph" in content -def test_debug_dot_print_with_options(sample_graphdef, dot_file): +def test_debug_dot_print_with_options(init_cuda, tmp_path): """debug_dot_print accepts GraphDebugPrintOptions.""" + sample_graphdef = GraphDefinition() + dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) @@ -1443,8 +1464,10 @@ def test_debug_dot_print_with_options(sample_graphdef, dot_file): assert dot_file.exists() -def test_debug_dot_print_invalid_options(sample_graphdef, dot_file): +def test_debug_dot_print_invalid_options(init_cuda, tmp_path): """debug_dot_print rejects invalid options type.""" + sample_graphdef = GraphDefinition() + dot_file = tmp_path / "graph.dot" _skip_if_no_mempool() with xfail_on_graph_mempool_oom(): sample_graphdef.allocate(ALLOC_SIZE) diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index c9ff492a2a8..fc82b7a04eb 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -685,6 +685,7 @@ def test_event_survives_graph_clone_and_execution(init_cuda): # ============================================================================= +@pytest.mark.thread_unsafe(reason="asserts cleanup on main thread") @pytest.mark.agent_authored(model="gpt-5.6") def test_user_object_cleanup_is_coalesced_on_python_thread(init_cuda): """More than 32 CUDA callbacks drain through one main-thread pending call.""" @@ -1412,6 +1413,7 @@ def test_memcpy_buffer_survives_close(init_cuda): assert list(out) == [0xCD] * 4 +@pytest.mark.thread_unsafe(reason="deferred cleanup on main thread which would wait") @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memcpy_buffer_allocations_released_after_graph_destroyed(init_cuda): """Destroying the graph frees both memcpy operand allocations. diff --git a/cuda_core/tests/graph/test_graph_memory_resource.py b/cuda_core/tests/graph/test_graph_memory_resource.py index 7b9c2aaf883..22757ca7556 100644 --- a/cuda_core/tests/graph/test_graph_memory_resource.py +++ b/cuda_core/tests/graph/test_graph_memory_resource.py @@ -5,7 +5,7 @@ """Tests for GraphMemoryResource allocation and attributes during graph capture.""" import pytest -from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer +from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer, thread_unsafe_on_windows from helpers.memory import xfail_on_graph_mempool_oom from cuda.core import ( @@ -21,6 +21,13 @@ from cuda.core.graph import GraphCompleteOptions from cuda_python_test_helpers import IS_WINDOWS, IS_WSL +# NOTE(seberg): "global" mode seems thread-unsafe even when working on stream +_GRAPH_MODES = [ + pytest.param("global", marks=pytest.mark.thread_unsafe(reason="gb instances share stream unsafely")), + "thread_local", + "relaxed", +] + def _common_kernels_alloc(): code = """ @@ -80,8 +87,9 @@ def free(self, buffers): self.stream.sync() -@pytest.mark.parametrize("mode", ["no_graph", "global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", ["no_graph"] + _GRAPH_MODES) @pytest.mark.parametrize("action", ["incr", "fill"]) +@thread_unsafe_on_windows def test_graph_alloc(mempool_device, mode, action): """Test basic graph capture with memory allocated and deallocated by GraphMemoryResource. @@ -130,7 +138,7 @@ def apply_kernels(mr, stream, out): assert compare_buffer_to_constant(out, 3) else: # Capture work, then upload and launch. - gb = device.create_graph_builder().begin_building(mode) + gb = stream.create_graph_builder().begin_building(mode) with xfail_on_graph_mempool_oom(device): apply_kernels(mr=gmr, stream=gb, out=out) graph = gb.end_building().complete() @@ -150,7 +158,8 @@ def apply_kernels(mr, stream, out): @pytest.mark.skipif(IS_WINDOWS or IS_WSL, reason="auto_free_on_launch not supported on Windows") -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) +@thread_unsafe_on_windows def test_graph_alloc_with_output(mempool_device, mode): """Test for memory allocated in a graph being used outside the graph.""" NBYTES = 64 @@ -168,7 +177,7 @@ def test_graph_alloc_with_output(mempool_device, mode): # Construct a graph to copy and increment the input. It returns a new # buffer allocated within the graph. The auto_free_on_launch option # is required to properly use the output buffer. - gb = device.create_graph_builder().begin_building(mode) + gb = stream.create_graph_builder().begin_building(mode) with xfail_on_graph_mempool_oom(device): out = gmr.allocate(NBYTES, stream=gb) out.copy_from(in_, stream=gb) @@ -195,7 +204,8 @@ def test_graph_alloc_with_output(mempool_device, mode): assert compare_buffer_to_constant(out, 6) -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) +@pytest.mark.thread_unsafe(reason="gb instances share default stream") def test_graph_mem_alloc_zero(mempool_device, mode): device = mempool_device gb = device.create_graph_builder().begin_building(mode) @@ -213,7 +223,8 @@ def test_graph_mem_alloc_zero(mempool_device, mode): assert buffer.device_id == int(device) -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) +@pytest.mark.thread_unsafe(reason="GMR is shared, so high mark is global") def test_graph_mem_set_attributes(mempool_device, mode): device = mempool_device stream = device.create_stream() @@ -265,7 +276,7 @@ def test_graph_mem_set_attributes(mempool_device, mode): mman.reset() -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) def test_gmr_check_capture_state(mempool_device, mode): """ Test expected errors (and non-errors) using GraphMemoryResource with graph @@ -284,7 +295,7 @@ def test_gmr_check_capture_state(mempool_device, mode): gmr.allocate(1, stream=stream) # Capturing - gb = device.create_graph_builder().begin_building(mode=mode) + gb = stream.create_graph_builder().begin_building(mode=mode) with xfail_on_graph_mempool_oom(device): gmr.allocate(1, stream=gb) # no error gb.end_building().complete() @@ -320,7 +331,7 @@ def test_graph_memory_resource_attributes_repr(mempool_device): assert "used_mem_high=" in r -@pytest.mark.parametrize("mode", ["global", "thread_local", "relaxed"]) +@pytest.mark.parametrize("mode", _GRAPH_MODES) def test_dmr_check_capture_state(mempool_device, mode): """ Test expected errors (and non-errors) using DeviceMemoryResource with graph @@ -334,7 +345,7 @@ def test_dmr_check_capture_state(mempool_device, mode): dmr.allocate(1, stream=stream).close() # no error # Capturing - gb = device.create_graph_builder().begin_building(mode=mode) + gb = stream.create_graph_builder().begin_building(mode=mode) with pytest.raises( RuntimeError, match=r"cannot perform memory operations on a capturing " diff --git a/cuda_core/tests/graph/test_graph_update.py b/cuda_core/tests/graph/test_graph_update.py index 54a04863cf4..c95e331913a 100644 --- a/cuda_core/tests/graph/test_graph_update.py +++ b/cuda_core/tests/graph/test_graph_update.py @@ -196,6 +196,7 @@ def new_callback(): assert called == ["new"] +@pytest.mark.thread_unsafe(reason="deferred cleanup on main thread which would wait") @pytest.mark.agent_authored(model="gpt-5.6") def test_failed_graph_update_does_not_adopt_attachments(init_cuda): """A rejected source graph keeps ownership separate from the exec.""" diff --git a/cuda_core/tests/helpers/buffers.py b/cuda_core/tests/helpers/buffers.py index aefb4f04193..2e71a2a199a 100644 --- a/cuda_core/tests/helpers/buffers.py +++ b/cuda_core/tests/helpers/buffers.py @@ -3,11 +3,13 @@ import ctypes +import pytest + from cuda.core import Buffer, Device, MemoryResource from cuda.core._stream import Stream_accept from cuda.core._utils.cuda_utils import driver, handle_return -from . import libc +from . import IS_WINDOWS, IS_WSL, libc __all__ = [ "DummyDeviceMemoryResource", @@ -18,9 +20,19 @@ "compare_equal_buffers", "make_instrumented_memory_resource", "make_scratch_buffer", + "thread_unsafe_on_windows", ] +def thread_unsafe_on_windows(func): + # Tests that use these buffers and access the memory on the host are + # thread-unsafe on windows. On windows the GPU must be fully quiescent for host + # access to be safe and with threaded tests that would require a barrier. + if IS_WINDOWS or IS_WSL: + return pytest.mark.thread_unsafe(reason="windows host-access unsafe while GPU is working") + return func + + class StubMemoryResource(MemoryResource): """Device-only memory resource for tests that supply a fake pointer.""" diff --git a/cuda_core/tests/helpers/latch.py b/cuda_core/tests/helpers/latch.py index c28fb222641..7702c5f617e 100644 --- a/cuda_core/tests/helpers/latch.py +++ b/cuda_core/tests/helpers/latch.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import ctypes diff --git a/cuda_core/tests/memory/test_managed_ops.py b/cuda_core/tests/memory/test_managed_ops.py index cb686026c7e..99053e1f8c9 100644 --- a/cuda_core/tests/memory/test_managed_ops.py +++ b/cuda_core/tests/memory/test_managed_ops.py @@ -365,6 +365,7 @@ def test_from_handle(self, init_cuda): finally: plain.close() + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_read_mostly_roundtrip(self, external_managed_buffer): buf = external_managed_buffer assert buf.read_mostly is False @@ -373,6 +374,7 @@ def test_read_mostly_roundtrip(self, external_managed_buffer): buf.read_mostly = False assert buf.read_mostly is False + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_preferred_location_roundtrip(self, location_ops_device, external_managed_buffer): device = location_ops_device buf = external_managed_buffer @@ -387,6 +389,7 @@ def test_preferred_location_roundtrip(self, location_ops_device, external_manage buf.preferred_location = None assert buf.preferred_location is None + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_preferred_location_roundtrip_host_numa(self, location_ops_device): """Host(numa_id=N) round-trips correctly on CUDA 13 builds.""" from cuda.core._utils.version import binding_version @@ -407,6 +410,7 @@ def test_preferred_location_roundtrip_host_numa(self, location_ops_device): finally: plain.close() + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_accessed_by_add_discard(self, location_ops_device, external_managed_buffer): device = location_ops_device buf = external_managed_buffer @@ -418,6 +422,7 @@ def test_accessed_by_add_discard(self, location_ops_device, external_managed_buf buf.accessed_by.discard(device) assert device not in buf.accessed_by + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_accessed_by_mutable_set_interface(self, location_ops_device, external_managed_buffer): """Full MutableSet conformance pass on AccessedBySetProxy. @@ -437,6 +442,7 @@ def test_accessed_by_mutable_set_interface(self, location_ops_device, external_m non_member=Host(numa_id=0), ) + @pytest.mark.thread_unsafe(reason="external_managed_buffer is shared between threads") def test_accessed_by_set_assignment(self, location_ops_device, external_managed_buffer): device = location_ops_device buf = external_managed_buffer diff --git a/cuda_core/tests/system/test_system_device.py b/cuda_core/tests/system/test_system_device.py index 028b7e7adf7..078ca5a835c 100644 --- a/cuda_core/tests/system/test_system_device.py +++ b/cuda_core/tests/system/test_system_device.py @@ -377,6 +377,7 @@ def test_c2c_mode_enabled(subtests): @pytest.mark.skipif(helpers.IS_WSL or helpers.IS_WINDOWS, reason="Persistence mode not supported on WSL or Windows") +@pytest.mark.thread_unsafe(reason="device persistence mode is global state") def test_persistence_mode_enabled(subtests): for device in system.Device.get_all_devices(): with subtests.test(device_index=device.index): diff --git a/cuda_core/tests/test_event.py b/cuda_core/tests/test_event.py index 3e765a9e0d0..bbe5a16efac 100644 --- a/cuda_core/tests/test_event.py +++ b/cuda_core/tests/test_event.py @@ -119,6 +119,7 @@ def test_error_timing_recorded(): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") +@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") def test_error_timing_incomplete(): device = Device() device.set_current() @@ -223,6 +224,7 @@ def test_event_ipc_descriptor_non_ipc(init_cuda): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") +@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") def test_event_is_done_false(init_cuda): """Event.is_done returns False when captured work has not yet completed.""" device = Device() diff --git a/cuda_core/tests/test_helpers.py b/cuda_core/tests/test_helpers.py index ed1e3bc6e74..8c55b924d77 100644 --- a/cuda_core/tests/test_helpers.py +++ b/cuda_core/tests/test_helpers.py @@ -6,7 +6,7 @@ import types import pytest -from helpers.buffers import PatternGen, compare_equal_buffers, make_scratch_buffer +from helpers.buffers import PatternGen, compare_equal_buffers, make_scratch_buffer, thread_unsafe_on_windows from helpers.latch import LatchKernel from helpers.logging import TimestampedLogger from helpers.oom_diagnostics import ( @@ -29,6 +29,7 @@ @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") +@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") def test_latchkernel(): """Test LatchKernel.""" log = TimestampedLogger(enabled=ENABLE_LOGGING) @@ -63,6 +64,7 @@ def test_latchkernel(): under_compute_sanitizer(), reason="Too slow under compute-sanitizer (UVM-heavy test).", ) +@thread_unsafe_on_windows def test_patterngen_seeds(): """Test PatternGen with seed argument.""" device = Device() @@ -81,6 +83,7 @@ def test_patterngen_seeds(): pgen.verify_buffer(buffer, seed=j) +@thread_unsafe_on_windows def test_patterngen_values(): """Test PatternGen with value argument, also compare_equal_buffers.""" device = Device() diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 99aaf938721..21f1ecab5ea 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -20,6 +20,7 @@ DummyUnifiedMemoryResource, StubMemoryResource, make_instrumented_memory_resource, + thread_unsafe_on_windows, ) from helpers.constants import POOL_SIZE from helpers.memory import ( @@ -283,9 +284,8 @@ def _pattern_bytes(value) -> bytes: @pytest.fixture(params=["device", "unified", "pinned"]) -def fill_env(request): - device = Device() - device.set_current() +def fill_env(request, init_cuda): + device = init_cuda if request.param == "device": mr = DummyDeviceMemoryResource(device) elif request.param == "unified": @@ -348,6 +348,7 @@ def fill_env(request): ) +@thread_unsafe_on_windows @pytest.mark.parametrize("value,size,exc", _FILL_CASES) def test_buffer_fill(fill_env, value, size, exc): device, mr = fill_env @@ -1519,6 +1520,8 @@ def test_managed_memory_resource_with_options(init_cuda): device.sync() dst_buffer.close() src_buffer.close() + # TODO(seberg): 2026-06: mr close may be unsafe with incomplete `buf.close()` + device.sync() def test_managed_memory_resource_preferred_location_default(init_cuda): diff --git a/cuda_core/tests/test_multiprocessing_warning.py b/cuda_core/tests/test_multiprocessing_warning.py index 1ddb53edb0f..2eb4aa55de9 100644 --- a/cuda_core/tests/test_multiprocessing_warning.py +++ b/cuda_core/tests/test_multiprocessing_warning.py @@ -12,6 +12,7 @@ import warnings from unittest.mock import patch +import pytest from helpers.constants import POOL_SIZE from cuda.core import DeviceMemoryResource, DeviceMemoryResourceOptions, EventOptions @@ -20,6 +21,9 @@ from cuda.core._memory._ipc import _reduce_allocation_handle from cuda.core._utils.cuda_utils import check_multiprocessing_start_method, reset_fork_warning +# We could move these to a (session) fixtures +pytestmark = pytest.mark.thread_unsafe(reason="all tests use unittest.mock.patch") + def test_warn_on_fork_method_device_memory_resource(ipc_device): """Test that warning is emitted when DeviceMemoryResource is pickled with fork method.""" diff --git a/cuda_core/tests/test_object_protocols.py b/cuda_core/tests/test_object_protocols.py index cd664501d06..b86cf5cf7e3 100644 --- a/cuda_core/tests/test_object_protocols.py +++ b/cuda_core/tests/test_object_protocols.py @@ -526,6 +526,25 @@ def sample_switch_node_alt(sample_graphdef): return sample_graphdef.switch(condition, 3) +# Resolve fixture names during pytest setup, before pytest-run-parallel starts +# worker threads. The workers then share the resolved, read-only test object. + + +@pytest.fixture +def sample_object(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def sample_object_a(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def sample_object_b(request): + return request.getfixturevalue(request.param) + + # ============================================================================= # Type groupings # ============================================================================= @@ -722,12 +741,11 @@ def sample_switch_node_alt(sample_graphdef): # ============================================================================= -@pytest.mark.parametrize("fixture_name", WEAKREF_TYPES) -def test_weakref_supported(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAKREF_TYPES, indirect=True) +def test_weakref_supported(sample_object): """Object supports weak references.""" - obj = request.getfixturevalue(fixture_name) - ref = weakref.ref(obj) - assert ref() is obj + ref = weakref.ref(sample_object) + assert ref() is sample_object # ============================================================================= @@ -735,27 +753,22 @@ def test_weakref_supported(fixture_name, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name", HASH_TYPES) -def test_hash_consistency(fixture_name, request): +@pytest.mark.parametrize("sample_object", HASH_TYPES, indirect=True) +def test_hash_consistency(sample_object): """Hash is consistent across multiple calls.""" - obj = request.getfixturevalue(fixture_name) - assert hash(obj) == hash(obj) + assert hash(sample_object) == hash(sample_object) -@pytest.mark.parametrize("a_name,b_name", SAME_TYPE_PAIRS) -def test_hash_distinct_same_type(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", SAME_TYPE_PAIRS, indirect=True) +def test_hash_distinct_same_type(sample_object_a, sample_object_b): """Distinct objects of the same type have different hashes.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert hash(obj_a) != hash(obj_b) # extremely unlikely + assert hash(sample_object_a) != hash(sample_object_b) # extremely unlikely -@pytest.mark.parametrize("a_name,b_name", list(itertools.combinations(HASH_TYPES, 2))) -def test_hash_distinct_cross_type(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", itertools.combinations(HASH_TYPES, 2), indirect=True) +def test_hash_distinct_cross_type(sample_object_a, sample_object_b): """Distinct objects of different types have different hashes.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert hash(obj_a) != hash(obj_b) # extremely unlikely + assert hash(sample_object_a) != hash(sample_object_b) # extremely unlikely # ============================================================================= @@ -763,41 +776,35 @@ def test_hash_distinct_cross_type(a_name, b_name, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name", EQ_TYPES) -def test_equality_basic(fixture_name, request): +@pytest.mark.parametrize("sample_object", EQ_TYPES, indirect=True) +def test_equality_basic(sample_object): """Object equality: reflexive, not equal to None or other types.""" - obj = request.getfixturevalue(fixture_name) - assert obj == obj - assert obj is not None - assert obj != "string" - if hasattr(obj, "handle"): - assert obj != obj.handle + assert sample_object == sample_object + assert sample_object is not None + assert sample_object != "string" + if hasattr(sample_object, "handle"): + assert sample_object != sample_object.handle -@pytest.mark.parametrize("a_name,b_name", list(itertools.combinations(EQ_TYPES, 2))) -def test_no_cross_type_equality(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", itertools.combinations(EQ_TYPES, 2), indirect=True) +def test_no_cross_type_equality(sample_object_a, sample_object_b): """No two distinct objects of different types should compare equal.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert obj_a != obj_b + assert sample_object_a != sample_object_b -@pytest.mark.parametrize("a_name,b_name", SAME_TYPE_PAIRS) -def test_same_type_inequality(a_name, b_name, request): +@pytest.mark.parametrize("sample_object_a,sample_object_b", SAME_TYPE_PAIRS, indirect=True) +def test_same_type_inequality(sample_object_a, sample_object_b): """Two distinct objects of the same type should not compare equal.""" - obj_a = request.getfixturevalue(a_name) - obj_b = request.getfixturevalue(b_name) - assert obj_a is not obj_b - assert obj_a != obj_b + assert sample_object_a is not sample_object_b + assert sample_object_a != sample_object_b -@pytest.mark.parametrize("fixture_name,copy_fn", FROM_HANDLE_COPIES) -def test_equality_same_handle(fixture_name, copy_fn, request): +@pytest.mark.parametrize("sample_object,copy_fn", FROM_HANDLE_COPIES, indirect=["sample_object"]) +def test_equality_same_handle(sample_object, copy_fn): """Two wrappers around the same handle should compare equal.""" - obj = request.getfixturevalue(fixture_name) - obj2 = copy_fn(obj) - assert obj == obj2 - assert hash(obj) == hash(obj2) + obj2 = copy_fn(sample_object) + assert sample_object == obj2 + assert hash(sample_object) == hash(obj2) # ============================================================================= @@ -805,48 +812,43 @@ def test_equality_same_handle(fixture_name, copy_fn, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name", DICT_KEY_TYPES) -def test_usable_as_dict_key(fixture_name, request): +@pytest.mark.parametrize("sample_object", DICT_KEY_TYPES, indirect=True) +def test_usable_as_dict_key(sample_object): """Object can be used as a dictionary key.""" - obj = request.getfixturevalue(fixture_name) - d = {obj: "value"} - assert d[obj] == "value" - assert obj in d + d = {sample_object: "value"} + assert d[sample_object] == "value" + assert sample_object in d -@pytest.mark.parametrize("fixture_name", DICT_KEY_TYPES) -def test_usable_in_set(fixture_name, request): +@pytest.mark.parametrize("sample_object", DICT_KEY_TYPES, indirect=True) +def test_usable_in_set(sample_object): """Object can be added to a set.""" - obj = request.getfixturevalue(fixture_name) - s = {obj} - assert obj in s + s = {sample_object} + assert sample_object in s -@pytest.mark.parametrize("fixture_name", WEAKREF_TYPES) -def test_usable_in_weak_value_dict(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAKREF_TYPES, indirect=True) +def test_usable_in_weak_value_dict(sample_object): """Object can be used as a WeakValueDictionary value.""" - obj = request.getfixturevalue(fixture_name) wvd = weakref.WeakValueDictionary() - wvd["key"] = obj - assert wvd["key"] is obj + wvd["key"] = sample_object + assert wvd["key"] is sample_object -@pytest.mark.parametrize("fixture_name", WEAK_KEY_TYPES) -def test_usable_in_weak_key_dict(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAK_KEY_TYPES, indirect=True) +def test_usable_in_weak_key_dict(sample_object): """Object can be used as a WeakKeyDictionary key.""" - obj = request.getfixturevalue(fixture_name) wkd = weakref.WeakKeyDictionary() - wkd[obj] = "value" - assert wkd[obj] == "value" + wkd[sample_object] = "value" + assert wkd[sample_object] == "value" -@pytest.mark.parametrize("fixture_name", WEAK_KEY_TYPES) -def test_usable_in_weak_set(fixture_name, request): +@pytest.mark.parametrize("sample_object", WEAK_KEY_TYPES, indirect=True) +def test_usable_in_weak_set(sample_object): """Object can be added to a WeakSet.""" - obj = request.getfixturevalue(fixture_name) ws = weakref.WeakSet() - ws.add(obj) - assert obj in ws + ws.add(sample_object) + assert sample_object in ws # ============================================================================= @@ -854,12 +856,10 @@ def test_usable_in_weak_set(fixture_name, request): # ============================================================================= -@pytest.mark.parametrize("fixture_name,pattern", REPR_PATTERNS) -def test_repr_format(fixture_name, pattern, request): +@pytest.mark.parametrize("sample_object,pattern", REPR_PATTERNS, indirect=["sample_object"]) +def test_repr_format(sample_object, pattern): """repr() returns a properly formatted string.""" - obj = request.getfixturevalue(fixture_name) - result = repr(obj) - assert re.fullmatch(pattern, result) + assert re.fullmatch(pattern, repr(sample_object)) # ============================================================================= @@ -868,10 +868,9 @@ def test_repr_format(fixture_name, pattern, request): @pytest.mark.parametrize("pickle_module", PICKLE_MODULES) -@pytest.mark.parametrize("fixture_name", PICKLE_TYPES) -def test_pickle_roundtrip(fixture_name, pickle_module, request): +@pytest.mark.parametrize("sample_object", PICKLE_TYPES, indirect=True) +def test_pickle_roundtrip(sample_object, pickle_module): """Object survives a pickle/cloudpickle roundtrip.""" mod = pytest.importorskip(pickle_module) - obj = request.getfixturevalue(fixture_name) - result = mod.loads(mod.dumps(obj)) - assert type(result) is type(obj) + result = mod.loads(mod.dumps(sample_object)) + assert type(result) is type(sample_object)