Merge upstream/main into feat/launch-config-sync-policy-2628

Sync PR #2637 with latest main (preserve branch history, no force-push).

Author: Omar Atie

.github/workflows/pr-metadata-check.yml

@@ -17,6 +17,9 @@ on:
       - reopened
       - ready_for_review
 
+permissions:
+  pull-requests: read
+
 jobs:
   check-metadata:
     name: PR has assignee, labels, and milestone

cuda_bindings/tests/nvml/test_pynvml.py

@@ -4,7 +4,6 @@
 # A set of tests ported from https://github.com/gpuopenanalytics/pynvml/blob/11.5.3/pynvml/tests/test_nvml.py
 
 import os
-import time
 
 import pytest
 
@@ -150,23 +149,6 @@ def test_device_get_power_usage(ngpus, handles, subtests):
             assert power_mwatts >= 0.0
 
 
-def test_device_get_total_energy_consumption(ngpus, handles, subtests):
-    for i in range(ngpus):
-        with subtests.test(device_index=i):
-            with unsupported_before(handles[i], None):
-                energy_mjoules1 = nvml.device_get_total_energy_consumption(handles[i])
-
-            for _ in range(10):  # idle for 150 ms
-                time.sleep(0.015)  # and check for increase every 15 ms
-                with unsupported_before(handles[i], None):
-                    energy_mjoules2 = nvml.device_get_total_energy_consumption(handles[i])
-                assert energy_mjoules2 >= energy_mjoules1
-                if energy_mjoules2 > energy_mjoules1:
-                    break
-            else:
-                raise AssertionError("energy did not increase across 150 ms interval")
-
-
 # [Skipping] pynvml.nvmlDeviceGetGpuOperationMode
 # [Skipping] pynvml.nvmlDeviceGetCurrentGpuOperationMode
 # [Skipping] pynvml.nvmlDeviceGetPendingGpuOperationMode

cuda_core/cuda/core/_program.pyi

@@ -313,6 +313,11 @@ class ProgramOptions:
         Load NVIDIA's `libdevice <https://docs.nvidia.com/cuda/libdevice-users-guide/>`_
         math builtins library. Only supported for the NVVM backend.
         Default: False
+    numba_debug : bool, optional
+        Emit the debug information layout expected by Numba. Recognized only by
+        newer toolkits; compilers that do not support it reject the option with
+        an error.
+        Default: False
     """
     name: str | None = 'default_program'
     arch: str | None = None
@@ -424,6 +429,17 @@ _nvvm_import_attempted = False
 def _can_load_generated_ptx() -> bool:
     """Check if the driver can load PTX generated by the current NVRTC version."""
 
+def _assert_single_dashed_nvvm_options(options: list[str]) -> None:
+    """Guard against emitting a double-dashed option to libNVVM.
+
+    libNVVM's parser accepts only single-dashed options and rejects the
+    double-dashed spelling of every option with NVVM_ERROR_INVALID_OPTION
+    (see #2570). Every option on this path is generated from typed fields, so
+    a double dash can only mean a bug in ``cuda.core`` rather than bad user
+    input. Fail here, naming the option, instead of leaving the user with
+    libNVVM's opaque error.
+    """
+
 def _program_compile_uncached(program, target_type, name_expressions, logs):
     """Run ``Program_compile`` without the cache wrapper.
 

cuda_core/cuda/core/_program.pyx

@@ -466,6 +466,11 @@ class ProgramOptions:
         Load NVIDIA's `libdevice <https://docs.nvidia.com/cuda/libdevice-users-guide/>`_
         math builtins library. Only supported for the NVVM backend.
         Default: False
+    numba_debug : bool, optional
+        Emit the debug information layout expected by Numba. Recognized only by
+        newer toolkits; compilers that do not support it reject the option with
+        an error.
+        Default: False
     """
 
     name: str | None = "default_program"
@@ -1220,6 +1225,24 @@ cdef inline list _prepare_nvrtc_options_impl(object opts):
     return [o.encode() for o in options]
 
 
+cpdef void _assert_single_dashed_nvvm_options(options: list[str]) except *:
+    """Guard against emitting a double-dashed option to libNVVM.
+
+    libNVVM's parser accepts only single-dashed options and rejects the
+    double-dashed spelling of every option with NVVM_ERROR_INVALID_OPTION
+    (see #2570). Every option on this path is generated from typed fields, so
+    a double dash can only mean a bug in ``cuda.core`` rather than bad user
+    input. Fail here, naming the option, instead of leaving the user with
+    libNVVM's opaque error.
+    """
+    for option in options:
+        if option.startswith("--"):
+            raise RuntimeError(
+                f"Internal error: NVVM option {option!r} is double-dashed. libNVVM accepts "
+                f"only single-dashed options; emit {option[1:]!r} instead."
+            )
+
+
 cdef inline object _prepare_nvvm_options_impl(object opts, bint as_bytes):
     """Build NVVM-specific compiler options."""
     options = []
@@ -1232,8 +1255,10 @@ cdef inline object _prepare_nvvm_options_impl(object opts, bint as_bytes):
     options.append(f"-arch={arch}")
     if opts.debug is not None and opts.debug:
         options.append("-g")
+    # libNVVM only accepts single-dashed options; the double-dashed spelling
+    # accepted by NVRTC is rejected with NVVM_ERROR_INVALID_OPTION.
     if opts.numba_debug:
-        options.append("--numba-debug")
+        options.append("-numba-debug")
     if opts.device_code_optimize is False:
         options.append("-opt=0")
     elif opts.device_code_optimize is True:
@@ -1313,6 +1338,8 @@ cdef inline object _prepare_nvvm_options_impl(object opts, bint as_bytes):
     if unsupported:
         raise CUDAError(f"The following options are not supported by NVVM backend: {', '.join(unsupported)}")
 
+    _assert_single_dashed_nvvm_options(options)
+
     if as_bytes:
         return [o.encode() for o in options]
     else:

cuda_core/docs/source/release/1.2.0-notes.rst

@@ -120,6 +120,16 @@ Fixes and enhancements
   ``coscheduled_sm_count`` explicitly when an aligned result is required.
   (`#2389 <https://github.com/NVIDIA/cuda-python/pull/2389>`__)
 
+- ``ProgramOptions(numba_debug=True)`` now works on the NVVM backend. The
+  option was emitted to libNVVM as ``--numba-debug``, but libNVVM accepts only
+  single-dashed options, so every such compile failed with
+  ``NVVM_ERROR_INVALID_OPTION``. It is now emitted as ``-numba-debug``, matching
+  what numba-cuda passes on the NVVM path. The NVRTC backend accepts both
+  spellings and was unaffected. The option itself is only recognized by newer
+  toolkits; libNVVM from CUDA 12.x does not support it under either spelling and
+  still reports ``NVVM_ERROR_INVALID_OPTION``.
+  (closes `#2570 <https://github.com/NVIDIA/cuda-python/issues/2570>`__)
+
 Deprecation Notices
 -------------------
 

cuda_core/tests/system/test_system_device.py

@@ -590,7 +590,10 @@ def test_clock(subtests):
                 with unsupported_before(device, None):
                     pstate = device.performance_state
 
-                min_, max_ = clock.get_min_max_clock_of_pstate_mhz(pstate)
+                # Individual queries may be unsupported for a clock domain even
+                # on newer devices.
+                with unsupported_before(device, None):
+                    min_, max_ = clock.get_min_max_clock_of_pstate_mhz(pstate)
                 assert isinstance(min_, int)
                 assert min_ >= 0
                 assert isinstance(max_, int)
@@ -601,7 +604,7 @@ def test_clock(subtests):
                 assert isinstance(max_mhz, int)
                 assert max_mhz >= 0
 
-                with unsupported_before(device, DeviceArch.KEPLER):
+                with unsupported_before(device, None):
                     current_mhz = clock.get_current_mhz()
                 assert isinstance(current_mhz, int)
                 assert current_mhz >= 0

cuda_core/tests/test_program.py

@@ -93,12 +93,16 @@ def _check_nvvm_arch(arch: str) -> bool:
 
 
 def _check_nvvm_supports_numba_debug() -> bool:
-    """Check if the installed libNVVM recognizes --numba-debug (CTK 13.2+)."""
+    """Check if the installed libNVVM recognizes -numba-debug.
+
+    libNVVM only accepts single-dashed options, so the double-dashed spelling
+    used by NVRTC is rejected by every libNVVM version.
+    """
     if not _has_check_nvvm_compiler_options():
         return False
     from cuda.bindings.utils import check_nvvm_compiler_options
 
-    return check_nvvm_compiler_options(["--numba-debug"])
+    return check_nvvm_compiler_options(["-numba-debug"])
 
 
 @pytest.fixture(scope="session")
@@ -762,18 +766,53 @@ def test_program_options_as_bytes_nvvm_unsupported_option():
 
 @nvvm_available
 def test_nvvm_program_options_as_bytes_numba_debug():
-    """numba_debug must be plumbed through to libNVVM as --numba-debug
-    (see #1287)."""
+    """numba_debug must be plumbed through to libNVVM as -numba-debug
+    (see #1287, #2570). libNVVM rejects the double-dashed spelling."""
     options = ProgramOptions(arch="sm_80", debug=True, numba_debug=True)
     nvvm_bytes = options.as_bytes("nvvm")
-    assert b"--numba-debug" in nvvm_bytes
+    assert b"-numba-debug" in nvvm_bytes
+    assert b"--numba-debug" not in nvvm_bytes
     assert b"-g" in nvvm_bytes
 
 
+@pytest.mark.agent_authored(model="claude-opus-5[1m]")
+def test_nvvm_options_reject_double_dash():
+    """The guard must name a double-dashed option rather than let libNVVM
+    reject it with an opaque error (see #2570)."""
+    from cuda.core._program import _assert_single_dashed_nvvm_options
+
+    _assert_single_dashed_nvvm_options(["-arch=compute_80", "-g", "-numba-debug"])
+
+    with pytest.raises(RuntimeError, match=r"--numba-debug.*double-dashed"):
+        _assert_single_dashed_nvvm_options(["-arch=compute_80", "--numba-debug"])
+
+
+@nvvm_available
+@pytest.mark.agent_authored(model="claude-opus-5[1m]")
+def test_nvvm_program_options_as_bytes_all_single_dashed():
+    """Every option cuda.core emits to libNVVM must be single-dashed, because
+    libNVVM rejects the double-dashed spelling of all of them (see #2570).
+    This covers every NVVM-supported field of ProgramOptions."""
+    options = ProgramOptions(
+        arch="sm_80",
+        debug=True,
+        numba_debug=True,
+        device_code_optimize=True,
+        ftz=True,
+        prec_sqrt=True,
+        prec_div=True,
+        fma=True,
+    )
+    nvvm_bytes = options.as_bytes("nvvm")
+    assert nvvm_bytes, "expected at least one emitted option"
+    offenders = [o for o in nvvm_bytes if o.startswith(b"--")]
+    assert not offenders, f"double-dashed options are rejected by libNVVM: {offenders}"
+
+
 @nvvm_available
 @pytest.mark.skipif(
     not _check_nvvm_supports_numba_debug(),
-    reason="installed libNVVM does not recognize --numba-debug (needs CTK 13.2+)",
+    reason="installed libNVVM does not recognize -numba-debug",
 )
 def test_nvvm_program_numba_debug(init_cuda, nvvm_ir):
     options = ProgramOptions(arch="sm_80", debug=True, numba_debug=True)