feat: cuda arch detection for shorter builds, build failure detection

1 files changed, 183 insertions, 0 deletions

diff --git a/app/tests/test_backends_audiocpp.py b/app/tests/test_backends_audiocpp.py
index 6f97438..4a8ee5f 100644
--- a/app/tests/test_backends_audiocpp.py
+++ b/app/tests/test_backends_audiocpp.py
@@ -1407,6 +1407,189 @@ class BuildAudiocppTests(unittest.TestCase):
         self.assertEqual(len(notices), 1)
         self.assertIn("No build script found", notices[0])
 
+    def test_cuda_build_appends_detected_arch_flag(self):
+        with patch.object(common, "run_console_subprocess",
+                          return_value=0) as run, \
+                patch.object(make_server.build, "detect_cuda_arch",
+                             return_value="86"):
+            make_server.build.build_audiocpp(self.checkout, "cuda")
+        argv = run.call_args[0][0]
+        self.assertIn("--cuda-arch", argv)
+        self.assertEqual(argv[argv.index("--cuda-arch") + 1], "86")
+
+    def test_tui_stall_notice_reports_the_hang(self):
+        emitted, emit = self._emit()
+        with patch.object(common, "LOG_DIR", self.log_dir), \
+                patch.object(common, "run_console_subprocess",
+                             return_value=124):
+            rc = make_server.build.build_audiocpp(self.checkout, "cuda",
+                                            emit=emit)
+        self.assertEqual(rc, 124)
+        log_text = self._log_files()[0].read_text(encoding="utf-8")
+        self.assertIn("stalled", log_text)
+        notices = common.drain_post_tui_notices()
+        self.assertEqual(len(notices), 1)
+        self.assertIn("stalled", notices[0])
+        self.assertIn("stopped", notices[0])
+
+    def test_tui_ptxas_failure_appends_detected_arch_guidance(self):
+        emitted, emit = self._emit()
+
+        def fake_run(argv, cwd=None, emit=None, cancel=None, **kwargs):
+            emit("ptxas fatal   : (C7907) Internal compiler error.")
+            return 1
+
+        with patch.object(common, "LOG_DIR", self.log_dir), \
+                patch.object(common, "run_console_subprocess",
+                             side_effect=fake_run), \
+                patch.object(make_server.build, "detect_cuda_arch",
+                             return_value="86"):
+            rc = make_server.build.build_audiocpp(self.checkout, "cuda",
+                                            emit=emit)
+        self.assertEqual(rc, 1)
+        notice = common.drain_post_tui_notices()[0]
+        self.assertIn("CUDA toolkit bug", notice)
+        self.assertIn("AUDIOCPP_CUDA_ARCH", notice)
+        self.assertIn("Detected arch for this machine: 86", notice)
+        # The undetected-GPU table is not needed when detection worked.
+        self.assertNotIn("Arch per GPU", notice)
+
+    def test_tui_ptxas_failure_lists_gpu_table_when_undetected(self):
+        emitted, emit = self._emit()
+
+        def fake_run(argv, cwd=None, emit=None, cancel=None, **kwargs):
+            emit("ptxas fatal   : (C7907) Internal compiler error.")
+            return 1
+
+        with patch.object(common, "LOG_DIR", self.log_dir), \
+                patch.object(common, "run_console_subprocess",
+                             side_effect=fake_run), \
+                patch.object(make_server.build, "detect_cuda_arch",
+                             return_value=None):
+            rc = make_server.build.build_audiocpp(self.checkout, "cuda",
+                                            emit=emit)
+        self.assertEqual(rc, 1)
+        notice = common.drain_post_tui_notices()[0]
+        self.assertIn("Arch per GPU", notice)
+        self.assertIn("3090", notice)
+        self.assertIn("4090", notice)
+
+    def test_tui_plain_failure_has_no_ptxas_guidance(self):
+        emitted, emit = self._emit()
+        with patch.object(common, "LOG_DIR", self.log_dir), \
+                patch.object(common, "run_console_subprocess",
+                             return_value=3):
+            rc = make_server.build.build_audiocpp(self.checkout, "cuda",
+                                            emit=emit)
+        notice = common.drain_post_tui_notices()[0]
+        self.assertIn("failed (exit code 3)", notice)
+        self.assertNotIn("AUDIOCPP_CUDA_ARCH", notice)
+
+
+class DetectCudaArchTests(unittest.TestCase):
+    """detect_cuda_arch: env override, nvidia-smi probe, None fallbacks."""
+
+    def setUp(self):
+        # Run every case without an AUDIOCPP_CUDA_ARCH leak from the
+        # developer's own shell.
+        clean = {k: v for k, v in os.environ.items()
+                 if k != make_server.build.CUDA_ARCH_ENV}
+        patcher = patch.dict(os.environ, clean, clear=True)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_env_override_wins_without_probing(self):
+        with patch.dict(os.environ,
+                        {make_server.build.CUDA_ARCH_ENV: "86"}), \
+                patch.object(common, "run_console_subprocess_quiet") as run:
+            self.assertEqual(make_server.build.detect_cuda_arch(), "86")
+        run.assert_not_called()
+
+    def test_env_override_multi_gpu_and_commas(self):
+        with patch.dict(os.environ,
+                        {make_server.build.CUDA_ARCH_ENV: "86, 89;75"}):
+            self.assertEqual(make_server.build.detect_cuda_arch(),
+                             "86;89;75")
+
+    def test_env_override_real_virtual_suffixes_allowed(self):
+        with patch.dict(os.environ,
+                        {make_server.build.CUDA_ARCH_ENV: "86-real"}):
+            self.assertEqual(make_server.build.detect_cuda_arch(), "86-real")
+
+    def test_invalid_env_override_ignored_and_probe_runs(self):
+        probe = MagicMock(returncode=0, stdout=b"8.6\n")
+        with patch.dict(os.environ,
+                        {make_server.build.CUDA_ARCH_ENV: "rtx"}), \
+                patch.object(common, "run_console_subprocess_quiet",
+                             return_value=probe) as run:
+            self.assertEqual(make_server.build.detect_cuda_arch(), "86")
+        self.assertEqual(run.call_args[0][0][0], "nvidia-smi")
+
+    def test_compute_caps_parsed_and_deduped(self):
+        probe = MagicMock(returncode=0, stdout=b"8.6\n8.6\n12.0\n")
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=probe):
+            self.assertEqual(make_server.build.detect_cuda_arch(), "86;120")
+
+    def test_nvidia_smi_failure_yields_none(self):
+        probe = MagicMock(returncode=1, stdout=b"")
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=probe):
+            self.assertIsNone(make_server.build.detect_cuda_arch())
+
+    def test_unparsable_output_yields_none(self):
+        probe = MagicMock(returncode=0,
+                          stdout=b"NVIDIA-SMI has failed because...\n")
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=probe):
+            self.assertIsNone(make_server.build.detect_cuda_arch())
+
+    def test_unstartable_probe_yields_none(self):
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=None):
+            self.assertIsNone(make_server.build.detect_cuda_arch())
+
+    def test_probe_is_bounded_by_a_timeout(self):
+        probe = MagicMock(returncode=0, stdout=b"8.6\n")
+        with patch.object(common, "run_console_subprocess_quiet",
+                          return_value=probe) as run:
+            make_server.build.detect_cuda_arch()
+        self.assertIsNotNone(run.call_args[1].get("timeout"))
+
+
+class CudaArchArgvTests(unittest.TestCase):
+    """_cuda_arch_argv: the --cuda-arch flags and their status line."""
+
+    def _argv(self, backend, arch, emit=None):
+        with patch.object(make_server.build, "detect_cuda_arch",
+                          return_value=arch):
+            return make_server.build._cuda_arch_argv(backend, emit=emit)
+
+    def test_cuda_build_gets_the_detected_arch(self):
+        self.assertEqual(self._argv("cuda", "86"),
+                         ["--cuda-arch", "86"])
+
+    def test_multi_gpu_arch_passed_verbatim(self):
+        self.assertEqual(self._argv("cuda", "86;89"),
+                         ["--cuda-arch", "86;89"])
+
+    def test_detection_failure_means_no_flag(self):
+        self.assertEqual(self._argv("cuda", None), [])
+
+    def test_non_cuda_backends_never_get_the_flag(self):
+        for backend in ("cpu", "vulkan", "hip"):
+            with self.subTest(backend=backend):
+                self.assertEqual(self._argv(backend, "86"), [])
+
+    def test_status_lines_report_the_outcome(self):
+        detected, emit = [], lambda line: detected.append(line)
+        self._argv("cuda", "86", emit=emit)
+        self.assertIn("CUDA architecture: 86", detected[0])
+        undetected, emit = [], lambda line: undetected.append(line)
+        self._argv("cuda", None, emit=emit)
+        self.assertIn("portable default list", undetected[0])
+        self.assertIn(make_server.build.CUDA_ARCH_ENV, undetected[0])
+
 
 class AudiocppUpdateTests(unittest.TestCase):
     """update: stop the server, refresh the checkout, rebuild when stale.