From 03ed68bafec56ef8bb690440d796c1a742042907 Mon Sep 17 00:00:00 2001 From: historia Date: Fri, 28 Aug 2026 18:44:32 -0400 Subject: feat(audio.cpp): if github binary download fails, automatically start building from source --- app/backends/audiocpp/prebuilt.py | 167 ++++++++++++++++++++++++++++++-------- 1 file changed, 135 insertions(+), 32 deletions(-) (limited to 'app/backends/audiocpp/prebuilt.py') diff --git a/app/backends/audiocpp/prebuilt.py b/app/backends/audiocpp/prebuilt.py index 28d60b5..13edbc0 100644 --- a/app/backends/audiocpp/prebuilt.py +++ b/app/backends/audiocpp/prebuilt.py @@ -24,8 +24,11 @@ a newer release instead of rebuilding from source. Everything downloads through urllib (no new dependencies); the GitHub API publishes a ``sha256:`` digest per asset, which is verified before -extraction. Files saved by urllib carry no macOS quarantine attribute, so -the ad-hoc-signed binaries run without Gatekeeper prompts. +extraction. When the API is rate-limited, the release tag is resolved from +the release page's redirect instead (no API quota) and the download +proceeds without verification (with a warning). Files saved by urllib +carry no macOS quarantine attribute, so the ad-hoc-signed binaries run +without Gatekeeper prompts. """ import hashlib @@ -49,6 +52,13 @@ from backends import common RELEASES_API_URL = \ "https://api.github.com/repos/0xShug0/audio.cpp/releases/latest" +# The release pages (NOT the API): following /releases/latest's redirect +# reveals the newest tag without spending any of the API's rate limit +# (60 requests/hour unauthenticated, shared per IP), and the asset +# download URLs are deterministic from the tag. +RELEASES_LATEST_URL = "https://github.com/0xShug0/audio.cpp/releases/latest" +RELEASES_DOWNLOAD_URL = "https://github.com/0xShug0/audio.cpp/releases/download" + # GitHub rejects API/HTTP requests without a User-Agent header. USER_AGENT = "tts-audiobook-generator (+https://github.com/0xShug0/audio.cpp)" @@ -68,6 +78,8 @@ _ASSET_RE = re.compile( _CUDART_RE = re.compile( r"^audio-v[^/]+-cudart-windows-x64-cuda(?P[a-z0-9.]+)\.zip$") +_TAG_RE = re.compile(r"/releases/tag/(v[^/?#]+)") + # Download progress reporting: emit a line at most every 4 MiB. _REPORT_STEP = 4 * 1024 * 1024 @@ -206,6 +218,52 @@ def fetch_latest_release(*, timeout: int = API_TIMEOUT) -> Optional[dict]: return None +def resolve_latest_tag(*, timeout: int = API_TIMEOUT) -> Optional[str]: + """The latest release tag, resolved WITHOUT the GitHub API. + + ``/releases/latest`` redirects to ``/releases/tag/vX.Y.Z``; reading + the final URL is an ordinary page hit that costs none of the API's + rate limit, so "already current" checks and rate-limited installs + never depend on the API. Returns None when the redirect cannot be + followed (offline, unexpected page shape). + """ + request = urllib.request.Request(RELEASES_LATEST_URL, + headers={"User-Agent": USER_AGENT}) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + url = response.geturl() or "" + except OSError: + return None + match = _TAG_RE.search(url) + return match.group(1) if match else None + + +def _variant_tokens(backend: Optional[str], *, + platform: Optional[str] = None, + machine: Optional[str] = None, + cuda_variant: Optional[str] = None + ) -> Optional[Tuple[str, str, str, Optional[str]]]: + """The (platform, arch, variant, cudart) tokens for this machine. + + Returns None when this platform/machine/backend combination has no + release asset naming (see ``prebuilt_supported``). The binary asset's + variant token carries a "cuda" prefix for CUDA builds + (``...-bin-windows-x64-cuda12.4.zip``) while the cudart archive uses + the bare version (``...-cudart-...-cuda12.4.zip``). + """ + token = _platform_token(platform) + arch = _arch_token(machine) + if token is None or arch is None or not prebuilt_supported( + backend, platform=platform): + return None + if token == "macos": + return token, arch, "metal", None + if backend == "cuda": + cuda = cuda_variant or _default_cuda_variant() + return token, arch, "cuda" + cuda, cuda + return token, arch, backend, None + + def select_assets(assets: List[dict], backend: Optional[str], *, platform: Optional[str] = None, machine: Optional[str] = None, @@ -218,23 +276,11 @@ def select_assets(assets: List[dict], backend: Optional[str], *, matches (unsupported platform, HIP, an unexpected arch). The CUDA variant defaults to the driver-appropriate one (``_default_cuda_variant``). """ - token = _platform_token(platform) - arch = _arch_token(machine) - if token is None or arch is None or not prebuilt_supported( - backend, platform=platform): + tokens = _variant_tokens(backend, platform=platform, machine=machine, + cuda_variant=cuda_variant) + if tokens is None: return None - variant: Optional[str] - cuda: Optional[str] = None - if token == "macos": - variant = "metal" - elif backend == "cuda": - # The binary asset names carry a "cuda" prefix on the variant - # token (``...-bin-windows-x64-cuda12.4.zip``); the cudart asset - # names use the bare version (``...-cudart-...-cuda12.4.zip``). - cuda = cuda_variant or _default_cuda_variant() - variant = "cuda" + cuda - else: - variant = backend + token, arch, variant, cuda = tokens main = _find_asset(assets, token, arch, variant) if main is None: return None @@ -246,6 +292,38 @@ def select_assets(assets: List[dict], backend: Optional[str], *, return main, extra +def synthesize_assets(backend: Optional[str], tag: str, *, + platform: Optional[str] = None, + machine: Optional[str] = None, + cuda_variant: Optional[str] = None + ) -> Optional[Tuple[dict, Optional[dict]]]: + """Asset entries built from the tag alone, without the GitHub API. + + The rate-limited fallback of ``install_prebuilt``: asset names and + ``releases/download//`` URLs are deterministic, but the sha256 + digests are not — callers download these WITHOUT checksum + verification and must say so. Same (main, extra) contract as + ``select_assets``. + """ + tokens = _variant_tokens(backend, platform=platform, machine=machine, + cuda_variant=cuda_variant) + if tokens is None: + return None + token, arch, variant, cuda = tokens + + def asset(name: str) -> dict: + return {"name": name, "size": None, "digest": None, + "browser_download_url": + f"{RELEASES_DOWNLOAD_URL}/{tag}/{name}"} + + ext = ".tar.gz" if token == "macos" else ".zip" + main = asset(f"audio-{tag}-bin-{token}-{arch}-{variant}{ext}") + extra = None + if token == "windows" and backend == "cuda": + extra = asset(f"audio-{tag}-cudart-windows-{arch}-cuda{cuda}.zip") + return main, extra + + def _find_asset(assets: List[dict], platform_token: str, arch: str, variant: str) -> Optional[dict]: for asset in assets: @@ -426,13 +504,19 @@ def install_prebuilt(audiocpp_dir: Path, backend: Optional[str], *, cuda_variant: Optional[str] = None) -> int: """Download and install the latest prebuilt audiocpp_server for BACKEND. - Fetches the latest release, picks this machine's asset(s), downloads - them to a temp directory, verifies the published sha256 digests, then - extracts into ``build//bin/`` (replacing any previous content — - including a source build there), syncs the checkout to the release - tag, and writes the ``prebuilt.json`` marker. Streaming output and - cancellation behave like the other install steps. Returns 0 on - success, 130 when cancelled, 1 on any failure. + Resolution degrades gracefully when GitHub's API is unavailable (its + unauthenticated limit is 60 requests/hour per IP): normally the API + supplies the release assets and their sha256 digests; when it is + rate-limited the newest tag is resolved from the release page's + redirect (no API quota), the asset names are synthesized, and the + download proceeds WITHOUT checksum verification (with a loud + warning). Only when neither path can resolve the release does the + install fail. The archive(s) are downloaded to a temp directory, + verified, then extracted into ``build//bin/`` (replacing any + previous content — including a source build there), the checkout is + synced to the release tag, and the ``prebuilt.json`` marker written. + Streaming output and cancellation behave like the other install + steps. Returns 0 on success, 130 when cancelled, 1 on any failure. """ say = emit if emit is not None else print bin_dir = install_dir(audiocpp_dir, backend) @@ -442,16 +526,35 @@ def install_prebuilt(audiocpp_dir: Path, backend: Optional[str], *, return 1 say("[INFO] Checking the latest audio.cpp release...") release = fetch_latest_release() - if release is None: - return 1 - tag = str(release.get("tag_name") or "?") pair = select_assets(release.get("assets") or [], backend, - cuda_variant=cuda_variant) - if pair is None: + cuda_variant=cuda_variant) if release else None + if release is not None and pair is None: + # API reachable but nothing matches: a hard mismatch (unsupported + # backend, renamed assets) — do not paper over it with a + # synthesized name that would just 404. print(f"[ERROR] No prebuilt audiocpp_server asset for " - f"{backend!r} in release {tag}; build from source instead.") + f"{backend!r} in release {release.get('tag_name')}; build " + "from source instead.") return 1 - main, extra = pair + if pair is not None: + tag = str(release.get("tag_name") or "?") + main, extra = pair + else: + tag = resolve_latest_tag() + if tag is None: + print("[ERROR] Could not reach GitHub to resolve the latest " + "audio.cpp release (rate limit or offline); try again " + "later or build from source.") + return 1 + pair = synthesize_assets(backend, tag, cuda_variant=cuda_variant) + if pair is None: + print(f"[ERROR] No prebuilt audiocpp_server asset exists for " + f"{backend!r} in release {tag}; build from source " + "instead.") + return 1 + main, extra = pair + say("[WARNING] GitHub's release API is unreachable (rate limit?) " + f"— installing {tag} WITHOUT checksum verification.") existing = installed_release(audiocpp_dir, backend) if existing and existing.get("tag") == tag \ and existing.get("asset") == main.get("name"): -- cgit v1.2.3