build: make Cython/extension artifacts CUDA-major aware

.github/workflows/ci-pixi-source-test.yml

@@ -15,6 +15,11 @@
 #   - build-smoke (PRs): CPU-only. Source-builds bindings + core, imports them,
 #     builds the cython test extensions and checks placement. Catches the
 #     compile / ABI / .so-placement regressions WITHOUT a GPU.
+#   - build-identity-roundtrip (nightly + manual): CPU-only. cu13 -> cu12 ->
+#     cu13 in one checkout, so build artifacts cannot be reused across CUDA
+#     majors. Kept off PRs to avoid adding another job to the org's shared
+#     concurrent-job quota; the fast unit tests in
+#     cuda_core/tests/test_build_hooks.py cover the same intent per-PR.
 #   - full-test (nightly + manual): GPU runner, full `pixi run test`.
 
 name: "CI: pixi run test (source build)"
@@ -108,6 +113,63 @@ jobs:
           done
           echo "cython test extensions placed correctly"
 
+  # ── Nightly guard: build artifacts must be CUDA-major aware ──
+  #
+  # Neither Cython nor setuptools tracks the CUDA major in its own up-to-date
+  # check: Cython does not hash `compile_time_env`, and an editable install's
+  # .so is named by the Python ABI tag alone. Before build_hooks keyed them,
+  # a cu13 build followed by a cu12 build in the same checkout failed while
+  # compiling cu13-generated C++ against CUDA 12 headers.
+  #
+  # The round trip (not just cu13 -> cu12) is what catches the second half:
+  # coming back to cu13 must not silently reuse the cu12 extension.
+  #
+  # cuda_core only: cuda_bindings cannot be source-built in its cu12
+  # environment at all, for reasons unrelated to stale artifacts.
+  build-identity-roundtrip:
+    name: "cu13 -> cu12 -> cu13 round trip (linux-64, CPU)"
+    if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.repository_owner == 'nvidia' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          # Full history + tags so setuptools-scm derives the real (13.x)
+          # package version; a shallow checkout yields 0.1.dev1, which trips
+          # cuda.core's "cuda.bindings 12.x or 13.x must be installed" guard.
+          fetch-depth: 0
+
+      - name: Setup pixi
+        # Pinned to a commit SHA; install logic lives in the action and is
+        # auditable/pinned (vs. a curl|bash of an unverified installer).
+        uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
+        with:
+          pixi-version: ${{ env.PIXI_VERSION }}
+          run-install: false
+
+      - name: Build cu13, then cu12, then cu13 again in one checkout
+        run: |
+          for cuda_env in cu13 cu12 cu13; do
+            echo "::group::${cuda_env}"
+            pixi run --manifest-path cuda_core -e "${cuda_env}" \
+              python -c "import cuda.core; print('core import OK')"
+            echo "::endgroup::"
+          done
+          # The last build was cu13, and each major must have kept its own
+          # generated sources rather than overwriting the other's.
+          stamp=$(cat cuda_core/build/.build-cuda-major)
+          if [ "${stamp}" != "13" ]; then
+            echo "::error::build stamp is '${stamp}', expected 13"
+            exit 1
+          fi
+          for major in cu12 cu13; do
+            if [ ! -d "cuda_core/build/cython/${major}" ]; then
+              echo "::error::no ${major} generated-source directory"
+              exit 1
+            fi
+          done
+
   # ── Nightly: full `pixi run test` on a GPU runner ──
   full-test:
     name: "pixi run test (${{ inputs.cuda-env || 'cu13' }}, linux-64, GPU)"

cuda_core/build_hooks.py

@@ -124,6 +124,55 @@ def _determine_cuda_major_version() -> str:
 # used later by setup()
 _extensions = None
 
+# Where per-configuration build artifacts live. Anchored to this file rather
+# than the cwd, since a project can be built from anywhere.
+_BUILD_DIR = Path(__file__).parent / "build"
+
+# Records the CUDA major of the last completed build, so setup.py can force
+# build_ext when it changes. Written by record_build_major().
+_BUILD_MAJOR_STAMP = _BUILD_DIR / ".build-cuda-major"
+
+force_build_ext = False
+
+
+def _check_build_major() -> str:
+    """Return the CUDA major to key build artifacts by, and set force_build_ext.
+
+    Cython's up-to-date check does not hash ``compile_time_env``, so generated
+    sources for one CUDA major would otherwise be reused for another. Keying
+    the generated-source directory fixes that, but not the compiled extension:
+    in an editable install it lands in the source tree under a name keyed by
+    the Python ABI tag alone, with nowhere to record the CUDA major. On a
+    cu12 -> cu13 -> cu12 round trip build_ext would find the older cu12
+    generated source next to the newer cu13 .so and skip the rebuild, so the
+    major is also stamped and build_ext forced whenever it changes.
+    """
+    global force_build_ext
+
+    cuda_major = _determine_cuda_major_version()
+    try:
+        previous = _BUILD_MAJOR_STAMP.read_text(encoding="utf-8").strip()
+    except FileNotFoundError:
+        previous = None
+
+    # A missing stamp means the last build's major is unknown, so force too.
+    # On a first build that costs nothing: there are no artifacts to reuse.
+    if previous != cuda_major:
+        print(f"CUDA major of last build: {previous} (building {cuda_major}); forcing a full rebuild")
+        force_build_ext = True
+
+    return cuda_major
+
+
+def record_build_major() -> None:
+    """Stamp the CUDA major of the build that just completed.
+
+    setup.py calls this after build_ext succeeds, so that a build which failed
+    partway through does not claim outputs it never produced.
+    """
+    _BUILD_MAJOR_STAMP.parent.mkdir(parents=True, exist_ok=True)
+    _BUILD_MAJOR_STAMP.write_text(_determine_cuda_major_version() + "\n", encoding="utf-8")
+
 
 def _build_cuda_core(debug=False):
     # Customizing the build hooks is needed because we must defer cythonization until cuda-bindings,
@@ -219,8 +268,13 @@ def get_sources(mod_name):
         for mod in module_names()
     )
 
+    # Deliberately after the cuda.bindings import above: this re-enters
+    # _get_cuda_path() and reads cuda.h, which must not run before the
+    # pathfinder import has repaired PEP 517 namespace shadowing.
+    cuda_major = _check_build_major()
+
     nthreads = int(os.environ.get("CUDA_PYTHON_PARALLEL_LEVEL", os.cpu_count() // 2))
-    compile_time_env = {"CUDA_CORE_BUILD_MAJOR": int(_determine_cuda_major_version())}
+    compile_time_env = {"CUDA_CORE_BUILD_MAJOR": int(cuda_major)}
     compiler_directives = {"embedsignature": True, "warn.deprecated.IF": False, "freethreading_compatible": True}
     _CythonOptions.warning_errors = True
     if COMPILE_FOR_COVERAGE:
@@ -229,7 +283,10 @@ def get_sources(mod_name):
         ext_modules,
         verbose=True,
         language_level=3,
-        build_dir="." if COMPILE_FOR_COVERAGE else "build/cython",
+        # CUDA_PYTHON_COVERAGE deliberately generates in-tree so the sources can
+        # be packaged; every other build gets its own per-configuration cache,
+        # anchored alongside the stamp so both resolve the same from any cwd.
+        build_dir="." if COMPILE_FOR_COVERAGE else str(_BUILD_DIR / "cython" / f"cu{cuda_major}"),
         nthreads=nthreads,
         compiler_directives=compiler_directives,
         compile_time_env=compile_time_env,

cuda_core/setup.py

@@ -51,6 +51,13 @@ def _build_aoti_shim_lib(compiler, plat_name):
 
 
 class build_ext(_build_ext):  # noqa: N801
+    def finalize_options(self):
+        super().finalize_options()
+        # A cu13 .so in the source tree looks perfectly fresh to a cu12 build;
+        # see build_hooks._check_build_major().
+        if build_hooks.force_build_ext:
+            self.force = True
+
     def _configure_windows_tensor_bridge(self):
         if os.name != "nt" or getattr(self.compiler, "compiler_type", None) != "msvc":
             return
@@ -74,6 +81,7 @@ def build_extensions(self):
         self.parallel = nthreads
         self._configure_windows_tensor_bridge()
         super().build_extensions()
+        build_hooks.record_build_major()
 
 
 class build_py(_build_py):  # noqa: N801
@@ -84,11 +92,14 @@ def finalize_options(self):
             self.package_data[""] += ["*.pxi", "*.pyx", "*.cpp"]
 
 
-setup(
-    ext_modules=build_hooks._extensions,
-    cmdclass={
-        "build_ext": build_ext,
-        "build_py": build_py,
-    },
-    zip_safe=False,
-)
+# Guarded so tests can import the command classes above. setuptools always
+# runs this file as __main__, so real builds are unaffected.
+if __name__ == "__main__":
+    setup(
+        ext_modules=build_hooks._extensions,
+        cmdclass={
+            "build_ext": build_ext,
+            "build_py": build_py,
+        },
+        zip_safe=False,
+    )

cuda_core/tests/test_build_hooks.py

@@ -19,6 +19,7 @@
 import builtins
 import importlib.util
 import os
+import sys
 import tempfile
 from pathlib import Path
 from unittest import mock
@@ -165,3 +166,148 @@ def test_missing_cuda_path_raises_error(self):
             pytest.raises(RuntimeError, match="CUDA_PATH or CUDA_HOME"),
         ):
             build_hooks._determine_cuda_major_version()
+
+
+@pytest.fixture
+def stamp(tmp_path, monkeypatch):
+    """Redirect the build stamp to a scratch path.
+
+    _BUILD_MAJOR_STAMP is anchored to build_hooks.py rather than the working
+    directory, so it has to be replaced outright; chdir would not move it, and
+    record_build_major() would write into the real source tree.
+    """
+    scratch = tmp_path / "build" / ".build-cuda-major"
+    monkeypatch.setattr(build_hooks, "_BUILD_MAJOR_STAMP", scratch)
+    monkeypatch.setattr(build_hooks, "force_build_ext", False)
+    build_hooks._get_cuda_path.cache_clear()
+    build_hooks._determine_cuda_major_version.cache_clear()
+    get_cuda_path_or_home.cache_clear()
+    monkeypatch.setenv("CUDA_CORE_BUILD_MAJOR", "13")
+    return scratch
+
+
+def _write_stamp(stamp, cuda_major):
+    stamp.parent.mkdir(parents=True, exist_ok=True)
+    stamp.write_text(cuda_major + "\n")
+
+
+class TestBuildMajorStamp:
+    """Tests for _check_build_major() and record_build_major()."""
+
+    def test_missing_stamp_forces_rebuild(self, stamp):
+        # No stamp means the last build's major is unknown, so rebuild.
+        assert build_hooks._check_build_major() == "13"
+        assert build_hooks.force_build_ext is True
+
+    def test_same_major_does_not_force(self, stamp):
+        _write_stamp(stamp, "13")
+        assert build_hooks._check_build_major() == "13"
+        assert build_hooks.force_build_ext is False
+
+    def test_changed_major_forces_rebuild(self, stamp):
+        _write_stamp(stamp, "12")
+        assert build_hooks._check_build_major() == "13"
+        assert build_hooks.force_build_ext is True
+
+    def test_record_writes_stamp(self, stamp):
+        build_hooks.record_build_major()
+        assert stamp.read_text().strip() == "13"
+
+
+def _capture_cythonize_build_dir(monkeypatch, cuda_major):
+    """Run the cythonize setup for one CUDA major and report its build_dir.
+
+    cythonize() is replaced, so nothing is generated or compiled: this only
+    observes which directory the build was about to write into.
+    """
+    captured = {}
+
+    def fake_cythonize(ext_modules, **kwargs):
+        captured.update(kwargs)
+        return []
+
+    # Builds resolve the CTK for include dirs; stub it so the test runs
+    # where no toolkit is installed (e.g. the wheels CI jobs).
+    monkeypatch.setattr(build_hooks, "_get_cuda_path", lambda: "/nonexistent-cuda")
+    monkeypatch.setattr(build_hooks, "cythonize", fake_cythonize)
+    monkeypatch.setenv("CUDA_CORE_BUILD_MAJOR", cuda_major)
+    build_hooks._determine_cuda_major_version.cache_clear()
+    # _build_cuda_core() globs cuda/core/**/*.pyx relative to the cwd.
+    monkeypatch.chdir(Path(__file__).parent.parent)
+    # It also prepends cuda_bindings/ to sys.path; swap in a copy so the
+    # mutation lands there and the real list is restored on teardown.
+    monkeypatch.setattr(sys, "path", list(sys.path))
+
+    build_hooks._build_cuda_core()
+    return Path(captured["build_dir"])
+
+
+class TestGeneratedSourceDirIsKeyed:
+    """Generated C++ must not be shared between CUDA majors.
+
+    Cython's up-to-date check does not hash compile_time_env, so without a
+    per-major directory a cu13 build's generated sources are handed to a cu12
+    compiler (and vice versa).
+    """
+
+    def test_majors_use_different_dirs(self, monkeypatch):
+        dir_12 = _capture_cythonize_build_dir(monkeypatch, "12")
+        dir_13 = _capture_cythonize_build_dir(monkeypatch, "13")
+
+        assert dir_12 != dir_13
+        assert dir_12.name == "cu12"
+        assert dir_13.name == "cu13"
+
+    def test_dir_is_anchored_not_relative_to_cwd(self, monkeypatch):
+        # Anchored to build_hooks.py, so it must agree with the stamp
+        # regardless of where the build was invoked from.
+        build_dir = _capture_cythonize_build_dir(monkeypatch, "13")
+
+        assert build_dir.is_absolute()
+        assert build_dir.parent.parent == build_hooks._BUILD_MAJOR_STAMP.parent
+
+
+def _load_setup_py(monkeypatch):
+    """Import setup.py for its command classes.
+
+    Importing rather than running is only possible because setup() is guarded
+    by __name__ == "__main__"; setuptools invokes the file as a script, so the
+    guard does not affect real builds.
+
+    setup.py does a bare ``import build_hooks``, which resolves to
+    cuda_bindings' copy if that directory is on sys.path. Pin cuda_core's, so
+    the flag the test sets is the one setup.py reads.
+    """
+    monkeypatch.setitem(sys.modules, "build_hooks", build_hooks)
+    setup_path = Path(__file__).parent.parent / "setup.py"
+    spec = importlib.util.spec_from_file_location("cuda_core_setup", setup_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class TestForceReachesBuildExt:
+    """The rebuild decision must actually be handed to setuptools.
+
+    _check_build_major() only sets a flag; if build_ext does not read it, a
+    stale extension is silently kept because its mtime looks newer than the
+    regenerated sources.
+    """
+
+    @staticmethod
+    def _finalized_build_ext(force_flag, monkeypatch):
+        from setuptools.dist import Distribution
+
+        setup_py = _load_setup_py(monkeypatch)
+        assert setup_py.build_hooks is build_hooks
+        monkeypatch.setattr(build_hooks, "force_build_ext", force_flag)
+
+        cmd = setup_py.build_ext(Distribution({"name": "cuda-core", "version": "0"}))
+        cmd.finalize_options()
+        return cmd
+
+    def test_flag_set_forces_rebuild(self, monkeypatch):
+        assert self._finalized_build_ext(True, monkeypatch).force
+
+    def test_flag_clear_leaves_default(self, monkeypatch):
+        assert not self._finalized_build_ext(False, monkeypatch).force