ci: publish each release to PyPI, npm and crates.io, and open the Homebrew formula PR

.github/workflows/ci.yml

@@ -5,6 +5,11 @@ on:
     branches: [main]
   pull_request:
 
+# Every job only reads the repository: its checkout, and rust-sys-abi's
+# release download.
+permissions:
+  contents: read
+
 # Self-hosted runner safety: a push only happens with write access, but a
 # pull_request can come from an external fork. Gate every job so PR runs execute
 # on the laptop runner ONLY for branches pushed into THIS repo (i.e. by someone
@@ -73,3 +78,18 @@ jobs:
         run: cargo test
         env:
           BASERT_LIB_DIR: ${{ github.workspace }}/build
+
+  packaging:
+    name: packaging scripts (tests)
+    # GitHub-hosted, so it needs no fork guard. linux-arm64 is one of the
+    # platforms the npm launcher tests run on.
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+      # publish.yml runs these scripts only after a release; their engine
+      # staging, link check and npm launcher are tested here on synthetic
+      # bundles.
+      - run: python3 tests/scripts/test_packaging.py

packaging/check_engine_links.py

@@ -0,0 +1,161 @@
+"""Check that a staged engine folder loads on its own.
+
+    python3 packaging/check_engine_links.py <engine-dir>
+    python3 packaging/check_engine_links.py --own-name <library>
+
+Wheels and npm tarballs cannot carry symlinks, so stage-engine.sh ships the
+engine library as real files: under the name the bindings load
+(libbaseRT.dylib / libbaseRT.so), and under the name the library records for
+itself (its install name, @rpath/libbaseRT.0.dylib, or its SONAME,
+libbaseRT.so.0), which is what a program linked against it asks for at run
+time. Every other libbaseRT name is dropped. This fails, naming each, if
+
+* any Mach-O or ELF file in the folder needs a library of ours the folder
+  does not hold: a libbaseRT of any name, or on macOS anything loaded through
+  @rpath, @loader_path or @executable_path; or
+* a library in the folder records a name for itself the folder does not
+  hold, so a program linked against it after installation (a Rust or C
+  consumer pointed at the folder with BASERT_LIB_DIR) would build and then
+  not start.
+
+System libraries (libSystem, libcuda, libcublas) are the machine's to
+provide. --own-name prints the name a library records for itself (nothing if
+it records none); stage-engine.sh uses it to place that copy.
+
+The load commands are read from the files themselves, not through otool or
+ldd, so one runner checks both platforms' bundles.
+"""
+
+from __future__ import annotations
+
+import struct
+import sys
+from pathlib import Path
+
+MH_MAGIC_64 = 0xFEEDFACF
+# LC_LOAD_DYLIB, LC_LOAD_WEAK_DYLIB, LC_REEXPORT_DYLIB, LC_LAZY_LOAD_DYLIB,
+# LC_LOAD_UPWARD_DYLIB: what a file needs. LC_ID_DYLIB is a library's own name.
+MACHO_LOADS = {0xC, 0x80000018, 0x8000001F, 0x20, 0x80000023}
+LC_ID_DYLIB = 0xD
+ELF_MAGIC = b"\x7fELF"
+PT_LOAD, PT_DYNAMIC = 1, 2
+DT_NULL, DT_NEEDED, DT_STRTAB, DT_SONAME = 0, 1, 5, 14
+
+
+def _cstr(data: bytes, off: int) -> str:
+    return data[off : data.index(b"\0", off)].decode()
+
+
+def macho_names(data: bytes) -> tuple[list[str], str | None]:
+    """(the libraries a Mach-O file loads, its own install name or None)."""
+    ncmds = struct.unpack_from("<I", data, 16)[0]
+    deps, own, off = [], None, 32
+    for _ in range(ncmds):
+        cmd, size = struct.unpack_from("<II", data, off)
+        if cmd in MACHO_LOADS or cmd == LC_ID_DYLIB:
+            name = _cstr(data, off + struct.unpack_from("<I", data, off + 8)[0])
+            if cmd == LC_ID_DYLIB:
+                own = name
+            else:
+                deps.append(name)
+        off += size
+    return deps, own
+
+
+def elf_names(data: bytes) -> tuple[list[str], str | None]:
+    """(an ELF file's DT_NEEDED libraries, its DT_SONAME or None)."""
+    if data[4] != 2 or data[5] != 1:
+        raise ValueError("not a 64-bit little-endian ELF")
+    phoff = struct.unpack_from("<Q", data, 0x20)[0]
+    phentsize, phnum = struct.unpack_from("<HH", data, 0x36)
+    loads, dynamic = [], None
+    for i in range(phnum):
+        p_type, _, p_offset, p_vaddr, _, p_filesz = struct.unpack_from("<IIQQQQ", data, phoff + i * phentsize)
+        if p_type == PT_LOAD:
+            loads.append((p_vaddr, p_offset, p_filesz))
+        elif p_type == PT_DYNAMIC:
+            dynamic = (p_offset, p_filesz)
+    if dynamic is None:
+        return [], None  # statically linked
+    needed, soname, strtab = [], None, None
+    for off in range(dynamic[0], dynamic[0] + dynamic[1], 16):
+        tag, val = struct.unpack_from("<qQ", data, off)
+        if tag == DT_NULL:
+            break
+        if tag == DT_NEEDED:
+            needed.append(val)
+        elif tag == DT_SONAME:
+            soname = val
+        elif tag == DT_STRTAB:
+            strtab = val
+    if not needed and soname is None:
+        return [], None
+    # DT_STRTAB is an address; the segment that maps it gives its file offset.
+    base = next((off + strtab - vaddr for vaddr, off, size in loads if vaddr <= strtab < vaddr + size), None)
+    if base is None:
+        raise ValueError("DT_STRTAB is outside every PT_LOAD segment")
+    return [_cstr(data, base + n) for n in needed], None if soname is None else _cstr(data, base + soname)
+
+
+def link_names(f: Path) -> tuple[list[str], str | None] | None:
+    """What a Mach-O or ELF file loads and the name it records for itself; None for any other file."""
+    data = f.read_bytes()
+    if len(data) >= 32 and struct.unpack_from("<I", data)[0] == MH_MAGIC_64:
+        return macho_names(data)
+    if data[:4] == ELF_MAGIC:
+        return elf_names(data)
+    return None
+
+
+def ours(dep: str) -> bool:
+    return dep.rsplit("/", 1)[-1].startswith("libbaseRT") or dep.startswith("@")
+
+
+def missing_links(engine: Path) -> list[tuple[str, str]]:
+    """(file, library) for each library of ours a file needs that `engine` lacks."""
+    missing = []
+    for f in sorted(engine.iterdir()):
+        if not f.is_file() or f.is_symlink():
+            continue
+        names = link_names(f)
+        for dep in names[0] if names else []:
+            name = dep.rsplit("/", 1)[-1]
+            if ours(dep) and not (engine / name).is_file():
+                missing.append((f.name, name))
+    return missing
+
+
+def missing_own_names(engine: Path) -> list[tuple[str, str]]:
+    """(library, name) for each library of ours whose own recorded name `engine` lacks."""
+    missing = []
+    for f in sorted(engine.iterdir()):
+        if not f.is_file() or f.is_symlink():
+            continue
+        names = link_names(f)
+        own = names[1] if names else None
+        if own is not None and ours(own) and not (engine / own.rsplit("/", 1)[-1]).is_file():
+            missing.append((f.name, own.rsplit("/", 1)[-1]))
+    return missing
+
+
+def main() -> None:
+    if len(sys.argv) == 3 and sys.argv[1] == "--own-name":
+        names = link_names(Path(sys.argv[2]))
+        if names and names[1]:
+            print(names[1].rsplit("/", 1)[-1])
+        return
+    if len(sys.argv) != 2:
+        sys.exit(__doc__)
+    engine = Path(sys.argv[1])
+    missing = missing_links(engine)
+    for f, name in missing:
+        print(f"{engine}: {f} needs {name}, which is not in the folder", file=sys.stderr)
+    unnamed = missing_own_names(engine)
+    for f, name in unnamed:
+        print(f"{engine}: {f} calls itself {name}, which is not in the folder: a program linked against it would not start", file=sys.stderr)
+    if missing or unnamed:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

packaging/crates/stage.py

@@ -0,0 +1,194 @@
+"""Stage the Rust crates for crates.io in a scratch folder.
+
+    python3 packaging/crates/stage.py <out-dir>
+
+Writes two cargo workspaces, each publishable with
+`cargo publish --workspace` (cargo 1.90+, which also orders the crates):
+
+  <out-dir>/cli       the basert CLI (base-convert/) and its 7 library crates
+  <out-dir>/bindings  the Rust bindings (bindings/rust/): baseRT-sys, baseRT
+
+Both source folders keep their manifests as they are for development, so the
+changes crates.io needs are made to copies here:
+
+* Crate names get a `basert-` prefix (`base-format` -> `basert-format`) so they
+  don't claim generic names on the shared registry. The CLI package becomes
+  `basert-cli`, since crates.io treats `basert` and the bindings' `baseRT` as
+  the same name. Dependency keys are unchanged (`package = "basert-format"`),
+  and each library keeps its name (`[lib] name = "base_format"`), so the code,
+  each crate's own tests and doctests included, still says `base_format::`.
+* Every crate gets the `description` crates.io requires and a repository link,
+  and each path dependency between them gets the exact version it is published at.
+* crates.io packs each crate on its own, so the CLI can't `include_str!` the
+  quant profiles from ../../../profiles. The profiles it embeds are copied into
+  the crate and the paths shortened. The public profiles and the engine header
+  that tests read from the workspace are copied in too, into the crates whose
+  tests read them (`cargo test` on a crate from crates.io); those tests look in
+  the crate first.
+* baseRT-sys carries the engine's C headers in its own include/baseRT/, which
+  its build script hands to dependents' build scripts as DEP_BASERT_INCLUDE.
+* Every crate carries the LICENSE and NOTICE.
+"""
+
+from __future__ import annotations
+
+import re
+import shutil
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+HERE = Path(__file__).resolve().parent
+REPOSITORY = "https://github.com/basecompute/baseRT"
+HOMEPAGE = "https://docs.basecompute.co"
+
+# Source crate name -> (crates.io name, description).
+CLI_CRATES = {
+    "base-format": ("basert-format", "Reader and writer for the BaseRT .base model format"),
+    "base-quant": ("basert-quant", "Quantization packing routines for BaseRT .base models"),
+    "base-awq": ("basert-awq", "AWQ (activation-aware weight quantization) calibration for BaseRT models"),
+    "base-readers": ("basert-readers", "GGUF, MLX-safetensors and HF-safetensors readers for the BaseRT converter"),
+    "base-arch": ("basert-arch", "Per-architecture tensor-name normalization for the BaseRT converter"),
+    "base-sign": ("basert-sign", "ed25519 signing and verification of BaseRT .base manifests"),
+    "base-hub": ("basert-hub", "Model hub for BaseRT: resolve, download and catalog .base models"),
+    "base-convert": ("basert-cli", "The basert CLI: pull, convert and run LLMs locally with BaseRT"),
+}
+BINDING_CRATES = ["baseRT-sys", "baseRT"]
+IGNORE = shutil.ignore_patterns("target", ".DS_Store")
+
+
+def fail(msg: str) -> None:
+    sys.exit(f"stage.py: {msg}")
+
+
+def sub_once(pattern: str, repl: str, text: str, what: str) -> str:
+    new, n = re.subn(pattern, repl, text, count=1, flags=re.M)
+    if n != 1:
+        fail(f"could not {what}")
+    return new
+
+
+def add_package_fields(manifest: Path, fields: dict[str, str]) -> None:
+    """Insert `key = value` lines right after [package] name, skipping keys already set."""
+    text = manifest.read_text()
+    pkg = text[: text.find("\n[", text.find("[package]") + 1)]
+    lines = "".join(
+        f"{k} = {v}\n" for k, v in fields.items() if not re.search(rf"^{re.escape(k)}(\.workspace)?\s*=", pkg, re.M)
+    )
+    text = sub_once(r"^(name = .*\n)", lambda m: m.group(1) + lines, text, f"find name in {manifest}")
+    manifest.write_text(text)
+
+
+def add_license(crate: Path) -> None:
+    for name in ("LICENSE", "NOTICE"):
+        shutil.copy(ROOT / name, crate / name)
+
+
+def stage_cli(out: Path) -> str:
+    ws = out / "cli"
+    shutil.copytree(ROOT / "base-convert", ws, ignore=IGNORE)
+    root_toml = ws / "Cargo.toml"
+    text = root_toml.read_text()
+    version = re.search(r'^\[workspace\.package\][^\[]*?^version = "([^"]+)"', text, re.M | re.S).group(1)
+
+    for old, (new, _) in CLI_CRATES.items():
+        if old == "base-convert":
+            continue  # the CLI itself; nothing depends on it
+        text = sub_once(
+            rf'^{re.escape(old)} = \{{ path = "(crates/{re.escape(old)})" \}}$',
+            rf'{old} = {{ package = "{new}", version = "={version}", path = "\1" }}',
+            text,
+            f"rewrite workspace dependency {old}",
+        )
+    root_toml.write_text(text)
+
+    for old, (new, description) in CLI_CRATES.items():
+        crate = ws / "crates" / old
+        manifest = crate / "Cargo.toml"
+        m = sub_once(rf'^name = "{re.escape(old)}"$', f'name = "{new}"', manifest.read_text(), f"rename {old}")
+        if (crate / "src" / "lib.rs").exists():
+            if re.search(r"^\[lib\]", m, re.M):
+                fail(f"{old} already has a [lib] section")
+            m += f'\n[lib]\nname = "{old.replace("-", "_")}"\n'
+        manifest.write_text(m)
+        add_package_fields(
+            manifest,
+            {"description": f'"{description}"', "repository": f'"{REPOSITORY}"', "homepage": f'"{HOMEPAGE}"'},
+        )
+        add_license(crate)
+
+    # The CLI's crate page gets the workspace README (what `basert` is).
+    cli = ws / "crates" / "base-convert"
+    shutil.copy(ws / "README.md", cli / "README.md")
+    add_package_fields(cli / "Cargo.toml", {"readme": '"README.md"'})
+
+    # Embedded profiles: copy each into the crate, then point include_str! at the copy.
+    for src in (cli / "src").rglob("*.rs"):
+        code = src.read_text()
+        for rel in sorted(set(re.findall(r'include_str!\("\.\./\.\./\.\./(profiles/[^"]+)"\)', code))):
+            dest = cli / rel
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy(ws / rel, dest)
+        src.write_text(code.replace('include_str!("../../../profiles/', 'include_str!("../profiles/'))
+
+    # What tests read from the workspace: base-quant's and the CLI's read the
+    # public profiles, and the CLI's the engine header (its ABI number).
+    for crate in ("base-quant", "base-convert"):
+        shutil.copytree(ws / "profiles", ws / "crates" / crate / "profiles", dirs_exist_ok=True)
+    (cli / "include" / "baseRT").mkdir(parents=True)
+    shutil.copy(ROOT / "include" / "baseRT" / "baseRT.h", cli / "include" / "baseRT" / "baseRT.h")
+
+    for src in (ws / "crates").rglob("*.rs"):
+        if re.search(r'include_(str|bytes)!\("\.\./\.\./\.\.', src.read_text()):
+            fail(f"{src.relative_to(ws)} still embeds a file from outside its crate")
+    return version
+
+
+def stage_bindings(out: Path, version: str) -> None:
+    ws = out / "bindings"
+    shutil.copytree(ROOT / "bindings" / "rust", ws, ignore=IGNORE)
+    (ws / "Cargo.toml").write_text(
+        "# Generated by packaging/crates/stage.py so `cargo publish --workspace`\n"
+        "# publishes baseRT-sys before baseRT.\n"
+        '[workspace]\nresolver = "2"\nmembers = ["baseRT-sys", "baseRT"]\n'
+    )
+    shutil.copytree(ROOT / "include" / "baseRT", ws / "baseRT-sys" / "include" / "baseRT")
+    for crate in BINDING_CRATES:
+        manifest = ws / crate / "Cargo.toml"
+        found = re.search(r'^version = "([^"]+)"', manifest.read_text(), re.M).group(1)
+        if found != version:
+            fail(f"{crate} is at {found} but the CLI workspace is at {version}")
+        shutil.copy(ws / "README.md", ws / crate / "README.md")
+        add_license(ws / crate)
+        add_package_fields(
+            manifest, {"repository": f'"{REPOSITORY}"', "homepage": f'"{HOMEPAGE}"', "readme": '"README.md"'}
+        )
+    manifest = ws / "baseRT" / "Cargo.toml"
+    manifest.write_text(
+        sub_once(
+            r'^baseRT-sys = \{ path = "\.\./baseRT-sys" \}$',
+            f'baseRT-sys = {{ version = "={version}", path = "../baseRT-sys" }}',
+            manifest.read_text(),
+            "pin baseRT-sys",
+        )
+    )
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        sys.exit(__doc__)
+    # Only the public source tree is published. The engine's private checkout,
+    # which has tools/sync_public.py, makes one with packaging/public-tree.sh.
+    if (ROOT / "tools" / "sync_public.py").exists():
+        fail("run this from the tree packaging/public-tree.sh builds, not the internal checkout")
+    out = Path(sys.argv[1]).resolve()
+    if out.exists():
+        fail(f"{out} already exists")
+    out.mkdir(parents=True)
+    version = stage_cli(out)
+    stage_bindings(out, version)
+    print(f"staged {version}: {out / 'cli'} and {out / 'bindings'}")
+
+
+if __name__ == "__main__":
+    main()