Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 126 additions & 1 deletion cuda_core/tests/graph/test_graph_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

import cuda.bindings
from cuda.core import Device, LaunchConfig, LegacyPinnedMemoryResource, Program, ProgramOptions, StreamOptions, launch
from cuda.core.graph import GraphBuilder, GraphCompleteOptions, GraphDefinition
from cuda.core.graph import Graph, GraphBuilder, GraphCompleteOptions, GraphDefinition
from cuda.core.graph._graph_builder import (
_capture_callback_with_tail_failure_for_testing,
)
Expand All @@ -32,6 +32,13 @@ def _wait_until(predicate, timeout=5.0):
time.sleep(0.02)


def _skip_if_conditional_handles_unsupported():
from cuda.core._utils.version import binding_version, driver_version

if driver_version() < (12, 3, 0) or binding_version() < (12, 3, 0):
pytest.skip("conditional handles require CUDA driver and bindings 12.3+")


def test_graph_is_building(init_cuda):
gb = Device().create_graph_builder()
assert gb.is_building is False
Expand Down Expand Up @@ -895,3 +902,121 @@ def test_pdl_same_stream_primary_secondary_overlap_via_graph(init_cuda):
"PDL (Programmatic Dependent Launch) graph overlap was not observed. "
"If this keeps xfailing in CI, manually re-check on a quiet Hopper+ GPU."
)


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_join_rejects_non_builder(init_cuda):
"""join() type-checks its arguments before looking at capture state."""
gb = Device().create_graph_builder()
with pytest.raises(TypeError, match="All arguments must be GraphBuilder"):
GraphBuilder.join(gb, object())


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_builder_cuda_stream_protocol(init_cuda):
"""The builder exports its underlying stream, and stops doing so once closed."""
gb = Device().create_graph_builder()
protocol = gb.__cuda_stream__()
assert protocol[0] == 0
assert int(protocol[1]) == int(gb.stream.handle)
gb.close()
with pytest.raises(RuntimeError, match="GraphBuilder has been closed"):
gb.__cuda_stream__()


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_end_building_requires_active_capture(init_cuda):
"""end_building() on a builder that never started capturing is rejected."""
gb = Device().create_graph_builder()
with pytest.raises(RuntimeError, match="Graph builder is not building"):
gb.end_building()


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_debug_dot_print_requires_finished_build(init_cuda, tmp_path):
"""debug_dot_print() needs a completed capture, both before and during building."""
gb = Device().create_graph_builder()
with pytest.raises(RuntimeError, match="Graph has not finished building"):
gb.debug_dot_print(str(tmp_path / "unfinished.dot"))
gb.begin_building()
try:
with pytest.raises(RuntimeError, match="Graph has not finished building"):
gb.debug_dot_print(str(tmp_path / "capturing.dot"))
finally:
gb.end_building()
gb.debug_dot_print(str(tmp_path / "finished.dot"))
assert (tmp_path / "finished.dot").exists()


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_callback_requires_active_capture(init_cuda):
"""callback() is rejected outside an active capture."""
gb = Device().create_graph_builder()
with pytest.raises(RuntimeError, match="Cannot add callback when graph is not being built"):
gb.callback(lambda: None)


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_create_condition_requires_active_capture(init_cuda):
"""create_condition() is rejected outside an active capture."""
_skip_if_conditional_handles_unsupported()
gb = Device().create_graph_builder()
with pytest.raises(RuntimeError, match="Cannot create a condition when graph is not being built"):
gb.create_condition()


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_embed_requires_finished_child_and_capturing_parent(init_cuda):
"""embed() rejects an unfinished child and a parent that is not capturing."""
parent = Device().create_graph_builder()
# embed() checks the child before the parent, so the child must already be
# ended for the parent guard to be the one that fires here.
child = Device().create_graph_builder().begin_building().end_building()
with pytest.raises(ValueError, match="Parent graph is not being built"):
parent.embed(child)

unfinished = Device().create_graph_builder().begin_building()
capturing = Device().create_graph_builder().begin_building()
try:
with pytest.raises(ValueError, match="Child graph has not finished building"):
capturing.embed(unfinished)
finally:
capturing.end_building()
unfinished.end_building()


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_graph_builder_and_graph_cannot_be_constructed_directly():
"""Both types are factory-only; the guards run before any CUDA call."""
with pytest.raises(NotImplementedError, match="directly creating"):
GraphBuilder()
with pytest.raises(RuntimeError, match="directly constructing"):
Graph()


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_graph_builder_close_ends_active_capture(init_cuda):
"""close() during capture ends it and hands the stream back usable."""
empty = compile_common_kernels().get_kernel("empty_kernel")
stream = Device().create_stream()
gb = stream.create_graph_builder().begin_building()
launch(gb, LaunchConfig(grid=1, block=1), empty)
assert gb.is_building
gb.close()
with pytest.raises(RuntimeError, match="has been closed"):
_ = gb.is_building
# Ending capture via close() must leave the stream usable.
launch(stream, LaunchConfig(grid=1, block=1), empty)
stream.sync()
stream.close()


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_if_then_requires_active_capture(init_cuda):
"""Conditional nodes cannot be added once capture has ended."""
_skip_if_conditional_handles_unsupported()
gb = Device().create_graph_builder().begin_building()
condition = try_create_condition(gb)
gb.end_building()
with pytest.raises(RuntimeError, match="Cannot add conditional node when not actively capturing"):
gb.if_then(condition)
30 changes: 30 additions & 0 deletions cuda_core/tests/graph/test_graph_definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
"""Tests for GraphDefinition topology, node types, instantiation, and execution."""

import ctypes
import gc
import sys
import weakref
from collections.abc import Callable
from dataclasses import dataclass, field

Expand Down Expand Up @@ -910,6 +912,34 @@ def test_alloc_peer_access(mempool_device_x2):
assert d1.device_id in node.peer_access


@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_alloc_memory_type_host(init_cuda):
"""HOST graph alloc nodes reconstruct memory_type from the driver, not the Python argument."""
_skip_if_no_mempool()
from cuda.core._utils.cuda_utils import CUDAError

g = GraphDefinition()
try:
with xfail_on_graph_mempool_oom():
node = g.allocate(ALLOC_SIZE, memory_type=GraphMemoryType.HOST)
except CUDAError as e:
if "CUDA_ERROR_NOT_SUPPORTED" in str(e):
pytest.skip("Driver does not support graph alloc memory_type='host'")
raise

expected_dptr = node.dptr
succ = node.record(Device().create_event())
node_ref = weakref.ref(node)
del node
gc.collect()
assert node_ref() is None
reconstructed = next(iter(succ.pred))
assert isinstance(reconstructed, AllocNode)
assert reconstructed.memory_type == GraphMemoryType.HOST
assert reconstructed.dptr == expected_dptr
assert reconstructed.dptr != 0


# =============================================================================
# Join API
# =============================================================================
Expand Down
90 changes: 89 additions & 1 deletion cuda_core/tests/graph/test_graph_definition_lifetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,15 +76,19 @@ def _wait_until(predicate, timeout=None, interval=0.02):
raise AssertionError(f"condition not satisfied within {timeout}s")


from cuda.core import Device, DeviceMemoryResource, EventOptions, Kernel, LaunchConfig
from cuda.core import Device, DeviceMemoryResource, EventOptions, Kernel, LaunchConfig, LegacyPinnedMemoryResource
from cuda.core._utils.cuda_utils import CUDAError
from cuda.core._utils.version import driver_version
from cuda.core.graph import (
ChildGraphNode,
ConditionalNode,
EventRecordNode,
EventWaitNode,
FreeNode,
GraphDefinition,
HostCallbackNode,
KernelNode,
MemcpyNode,
)


Expand Down Expand Up @@ -1213,6 +1217,90 @@ def test_kernel_node_reconstruction_preserves_validity(init_cuda):
stream.sync()


def _pred_chain_memcpy(g, bufs):
memory_resource = LegacyPinnedMemoryResource()
src = memory_resource.allocate(8)
dst = memory_resource.allocate(8)
bufs.extend((src, dst))
node = g.memcpy(dst, src, 8)
src_ptr, dst_ptr, size = node.src, node.dst, node.size
succ = node.record(Device().create_event())

def check(reconstructed):
assert isinstance(reconstructed, MemcpyNode)
assert reconstructed.src == src_ptr
assert reconstructed.dst == dst_ptr
assert reconstructed.size == size

return node, succ, check


def _pred_chain_event_record(g, bufs):
event = Device().create_event()
node = g.record(event)
succ = node.wait(Device().create_event())

def check(reconstructed):
assert isinstance(reconstructed, EventRecordNode)
assert reconstructed.event.handle == event.handle

return node, succ, check


def _pred_chain_event_wait(g, bufs):
wait_event = Device().create_event()
node = g.wait(wait_event)
succ = node.record(Device().create_event())

def check(reconstructed):
assert isinstance(reconstructed, EventWaitNode)
assert reconstructed.event.handle == wait_event.handle

return node, succ, check


def _pred_chain_free(g, bufs):
_skip_if_no_mempool()
with xfail_on_graph_mempool_oom():
alloc = g.allocate(64)
node = alloc.deallocate(alloc.dptr)
free_dptr = node.dptr
succ = node.record(Device().create_event())

def check(reconstructed):
assert isinstance(reconstructed, FreeNode)
assert reconstructed.dptr == free_dptr

return node, succ, check


@pytest.mark.parametrize(
"factory",
[
_pred_chain_memcpy,
_pred_chain_event_record,
_pred_chain_event_wait,
_pred_chain_free,
],
ids=["memcpy", "event_record", "event_wait", "free"],
)
@pytest.mark.agent_authored(model="gpt-5.6-sol")
def test_graph_nodes_reconstructed_via_pred_chain(init_cuda, factory):
"""Dropping the original Python node forces `_create_from_driver` on pred walk."""
g = GraphDefinition()
bufs = []
try:
node, succ, check = factory(g, bufs)
node_ref = weakref.ref(node)
del node
_wait_until(lambda: node_ref() is None)
reconstructed = next(iter(succ.pred))
check(reconstructed)
finally:
for buf in bufs:
buf.close()


# =============================================================================
# Kernel argument lifetime — kernel nodes should keep argument objects alive
# =============================================================================
Expand Down
Loading
Loading