From a69828e4cc64d3046d9d6ae1163ba18e145ad723 Mon Sep 17 00:00:00 2001 From: baseRT sync Date: Tue, 27 Oct 2026 16:35:49 +1100 Subject: [PATCH] sync: open-core source for 0.4.2 Generated by tools/sync_public.py from the internal tree at 103edf28f. Only allow-listed paths are included; tuned quant profiles and internal tooling are excluded by rule. --- base-convert/Cargo.lock | 16 +- base-convert/Cargo.toml | 2 +- base-convert/README.md | 5 + base-convert/crates/base-arch/src/dflash.rs | 11 +- base-convert/crates/base-convert/src/hub.rs | 679 +++++++++++++++++- base-convert/crates/base-convert/src/main.rs | 109 ++- .../crates/base-convert/src/superfluid.rs | 4 +- .../base-convert/tests/whisper_quant.rs | 8 +- base-convert/crates/base-hub/src/registry.rs | 3 +- base-convert/crates/base-quant/src/profile.rs | 255 +------ bindings/node/README.md | 8 +- bindings/node/package.json | 2 +- bindings/node/src/index.ts | 33 +- bindings/python/README.md | 7 +- bindings/python/baseRT/__init__.py | 28 +- bindings/python/setup.py | 2 +- bindings/python/tests/test_baseRT.py | 42 +- bindings/rust/README.md | 23 +- bindings/rust/baseRT-sys/Cargo.toml | 6 +- bindings/rust/baseRT-sys/build.rs | 126 +++- bindings/rust/baseRT/Cargo.toml | 2 +- bindings/rust/baseRT/build.rs | 20 +- .../swift/Sources/CBaseRT/include/baseRT.h | 2 +- include/baseRT/baseRT.h | 2 +- packaging/npm/build-packages.mjs | 13 +- tests/scripts/test_packaging.py | 100 ++- 26 files changed, 1157 insertions(+), 351 deletions(-) diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index d7a4d57..bc9533e 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -151,7 +151,7 @@ dependencies = [ [[package]] name = "base-arch" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base-format", @@ -162,7 +162,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base-format", @@ -176,7 +176,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base-arch", @@ -201,7 +201,7 @@ dependencies = [ [[package]] name = "base-format" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "bitflags", @@ -217,7 +217,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base-format", @@ -236,7 +236,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base-format", @@ -248,7 +248,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base-format", @@ -262,7 +262,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.4.1" +version = "0.4.2" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index 87f7ffc..c71f5c7 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.4.1" +version = "0.4.2" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/base-convert/README.md b/base-convert/README.md index 56ea13d..27416f6 100644 --- a/base-convert/README.md +++ b/base-convert/README.md @@ -42,6 +42,11 @@ basert pull basecompute/ basert pull meta-llama/Llama-3.2-1B basert pull Qwen/Qwen3-0.6B --target base-q8 +# A variant by the name `basert list` gives it (`default-q8`, `cuda-q4mix`, +# a `file-…` variant), or a quant; the form `basert chat` and `serve` take: +basert pull basecompute/:default-q8 +basert pull Qwen/Qwen3-0.6B:q8 + basert pull --dry-run # resolve + print the plan, download nothing basert list # installed models (table; --json for JSON) basert list --remote # also show catalog models not yet installed diff --git a/base-convert/crates/base-arch/src/dflash.rs b/base-convert/crates/base-arch/src/dflash.rs index 0358b7f..f3f0517 100644 --- a/base-convert/crates/base-arch/src/dflash.rs +++ b/base-convert/crates/base-arch/src/dflash.rs @@ -12,9 +12,14 @@ //! `DFlash2DraftModel`, `Qwen3DSparkModel`) — converting it as a Qwen3 would //! drop `fc` / `hidden_norm` and write a broken LM. //! -//! Tensor policy: every tensor stays f16 (the drafter's acceptance rate is -//! the whole point of the scheme, and the runtime's block step is bound by -//! the target's verify anyway). `fc.weight` is split by input column into +//! Tensor policy: f16 by default; under base-convert's `--quantize-drafter` +//! the transformer layers and the `fc.{i}` taps follow the target scheme +//! (`--keep-drafter-fc-f16` leaves the taps f16), while norms, the DFlash 2 +//! selector codebooks and the Markov / confidence heads stay f16. The +//! catalog's default-q4 drafters quantize the layers; `basert pull` converts +//! a sidecar that way, the taps included for this family only, where 4-bit +//! taps were measured (DFlash2, 2026-10-19: acceptance unchanged), not for +//! DSpark or EAGLE-3. `fc.weight` is split by input column into //! one `fc.{i}.weight` per tap (base-convert's `FcSplitProvider`), so the //! runtime runs one GEMM per tap block of the tap-major input. //! diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index 999d2ff..c4f18ee 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -406,9 +406,6 @@ fn auto_pull_and_resolve( want_variant: Option<&str>, ) -> Result { let pull_id = preconverted_id(reg, id); - let target = want_variant - .map(|v| target_from_quant(&quant_tag(v))) - .unwrap_or(TargetScheme::BaseQ4); if pull_id != id { eprintln!("{id}: not installed — using pre-converted {pull_id}"); @@ -416,10 +413,13 @@ fn auto_pull_and_resolve( eprintln!("{id}: not installed — fetching (download + convert if needed)"); } + // The variant goes with the id: the pull `basert pull :` + // makes, so a catalog variant or a `file-…` one is fetched as named, not + // as the quant its last segment happens to spell. cmd_pull(PullArgs { - id: pull_id.clone(), + id: want_variant.map_or_else(|| pull_id.clone(), |v| format!("{pull_id}:{v}")), profile: None, - target, + target: TargetScheme::BaseQ4, revision: "main".to_string(), file: None, force: false, @@ -428,10 +428,11 @@ fn auto_pull_and_resolve( .with_context(|| format!("auto-fetching {pull_id}"))?; // Re-scan and return the freshly-installed artifact (registered under the - // pulled id, which may differ from the requested one). + // pulled id, which may differ from the requested one): the variant as + // named, or for a quant (`:q8`) the slot it installed under. let reg2 = MergedRegistry::load()?; if let Some(v) = want_variant { - if let Some(p) = reg2.local.installed_path(&pull_id, v) { + if let Some(p) = installed_best_variant(®2, &pull_id, v) { return Ok(p); } } @@ -673,11 +674,261 @@ const RUNTIME_COMMANDS: [&str; 5] = ["chat", "complete", "bench", "transcribe", pub fn cmd_pull(args: PullArgs) -> Result<()> { let reg = MergedRegistry::load()?; + if let Some((id, variant)) = args.id.split_once(':') { + let fetcher = HfFetcher::new(cache::hf_staging_dir(®.root))?; + return pull_variant(®, &args, id, variant, &fetcher); + } + pull_id(®, &args) +} + +/// Where `basert pull :` gets the variant from (see +/// [`variant_source`]). +#[derive(Debug, PartialEq)] +enum VariantSource { + /// The catalog row the variant names, pulled as a plain pull of it is. + Catalog(ModelRef), + /// A `.base` in the id's repo that no catalog row publishes, pulled as + /// `--file` would pull it. + File(String), + /// A quant, pulled as `--target` would pull it. + Quant(TargetScheme), +} + +/// What a pull says of a variant installed already. +fn say_installed(id: &str, variant: &str, path: &Path, dry_run: bool) { + if dry_run { + println!( + "plan: {id} [{variant}] already installed at {}", + path.display() + ); + } else { + eprintln!( + "{id} [{variant}] already installed: {}\n (use --force to re-pull)", + path.display() + ); + } +} + +/// `basert pull :`: the variant as `basert list` names it, the +/// form `basert chat` and `serve` already take. An installed variant is left +/// as it is (no request to the Hub); otherwise it is pulled from where +/// [`variant_source`] says. `--file`, `--profile` and `--target` each name what +/// the variant names, so none goes with one. +fn pull_variant( + reg: &MergedRegistry, + args: &PullArgs, + id: &str, + variant: &str, + fetcher: &dyn Fetcher, +) -> Result<()> { + if variant.is_empty() { + bail!( + "empty variant after ':' in `{}` — name a variant (e.g. `{id}:default-q8`) \ + or drop the colon. Run `basert list` to see installed variants.", + args.id + ); + } + let given = [ + args.file.as_ref().map(|_| "--file"), + args.profile.as_ref().map(|_| "--profile"), + (args.target != TargetScheme::BaseQ4).then_some("--target"), + ]; + if let Some(flag) = given.into_iter().flatten().next() { + bail!( + "`{}` names the variant, and so does {flag}: give one", + args.id + ); + } + if !args.force { + if let Some(path) = reg.local.installed_path(id, variant) { + say_installed(id, variant, &path, args.dry_run); + return Ok(()); + } + } + let pull = |file: Option, target: TargetScheme| PullArgs { + id: id.to_string(), + profile: None, + target, + revision: args.revision.clone(), + file, + force: args.force, + dry_run: args.dry_run, + }; + match variant_source(reg, id, variant, &args.revision, fetcher)? { + VariantSource::Catalog(r) => pull_catalog_row(reg, args, &r, fetcher), + VariantSource::File(file) => { + let named = pull(Some(file.clone()), args.target); + pull_named_file(reg, &named, &file, fetcher) + } + VariantSource::Quant(target) => pull_id(reg, &pull(None, target)), + } +} + +/// The catalog row a variant named, pulled as a plain pull of it is: left as +/// it is when installed, else its file from its own repo and revision, +/// checked against its checksums. The row is carried through, never looked +/// up again by its file name, which other rows of the id may share. +fn pull_catalog_row( + reg: &MergedRegistry, + args: &PullArgs, + r: &ModelRef, + fetcher: &dyn Fetcher, +) -> Result<()> { + let ModelRef::Catalog { id, variant, .. } = r else { + unreachable!("pull_catalog_row called with non-catalog ref"); + }; + if !args.force { + if let Some(path) = reg.local.installed_path(id, variant) { + say_installed(id, variant, &path, args.dry_run); + return Ok(()); + } + } + if args.dry_run { + print_plan(r, variant); + return Ok(()); + } + pull_catalog(®.root, r, fetcher) +} + +/// The `--target` a variant spells, taken whole: a quant tag (`q8`, `bf16`), +/// bare or under the prefix a converted variant or a `--target` value carries +/// (`default-q8`, `base-q8`, `base_q8`). `None` for anything else, so a name +/// that only ends in a quant (`default-q8-custom`, `typo-q8`) is not one. +fn target_alias(variant: &str) -> Option { + let v = variant.to_ascii_lowercase(); + let tag = ["default-", "base-", "base_"] + .iter() + .find_map(|p| v.strip_prefix(p)) + .unwrap_or(&v); + is_quant_tag(tag).then(|| target_from_quant(tag)) +} + +/// Where the variant `variant` of `id` comes from, in this order: a catalog +/// row of the id by that name (`default-q8`, `cuda-q4mix`), as `basert list +/// --remote` shows it, refused when it is for another backend; the repo file +/// a `file-…` variant was named for, looked for where `--file` looks; for a +/// width named on its own (`q8`, `f16`), the catalog's row of it for this +/// client; a quant `--target` takes, spelled as [`target_alias`] reads it. +/// Anything else, a name that merely contains a width included, is an error +/// naming the catalog's variants. +fn variant_source( + reg: &MergedRegistry, + id: &str, + variant: &str, + revision: &str, + fetcher: &dyn Fetcher, +) -> Result { + // A row this client can run before one it can't, which is refused + // saying which backend it is for. + let mut rows: Vec<&CatalogEntry> = reg + .catalog + .rows(id) + .filter(|e| e.quant == variant) + .collect(); + rows.sort_by_key(|e| !CatalogRegistry::backend_ok(e)); + if let Some(e) = rows.first() { + if !CatalogRegistry::backend_ok(e) { + bail!( + "{variant} is the {} bundle of {}; this client runs {}", + e.backend.as_deref().unwrap_or_default(), + e.id, + CatalogRegistry::client_backend() + ); + } + return Ok(VariantSource::Catalog(CatalogRegistry::entry_to_ref(e))); + } + if variant.starts_with(NAMED_VARIANT_PREFIX) { + return named_variant_file(reg, id, variant, revision, fetcher).map(VariantSource::File); + } + // A width named on its own: the catalog's row of it for this client, + // native before universal, as a plain pull picks one. The whole name + // must be the width; a `file-…` one can spell one (`file-model-q4-…`). + if base_hub::registry::quant_bits(variant) == Some(variant) { + match reg.catalog.resolve_with_status(id, Some(variant)) { + (Some(r), _) => return Ok(VariantSource::Catalog(r)), + (None, true) => bail!( + "{id} has no {variant} bundle for the {} backend this client runs", + CatalogRegistry::client_backend() + ), + (None, false) => {} + } + } + if let Some(target) = target_alias(variant) { + return Ok(VariantSource::Quant(target)); + } + let listed: Vec = reg + .catalog + .rows(id) + .filter(|e| CatalogRegistry::backend_ok(e)) + .map(|e| e.quant.clone()) + .collect(); + let listed = if listed.is_empty() { + String::new() + } else { + format!("; the catalog lists {}", listed.join(", ")) + }; + bail!( + "{id} has no variant {variant:?}{listed}. Name one `basert list --remote` shows, \ + a quant (q4, q8, bf16, …), or a file with --file" + ) +} + +/// The repo file the `file-…` variant `variant` was named for (see +/// [`named_variant`]), looked for where `--file` looks: the repos the id's +/// catalog rows use, else the id's own. A file the catalog publishes installs +/// under its row's variant, never a `file-…` one, so it is not that file. +fn named_variant_file( + reg: &MergedRegistry, + id: &str, + variant: &str, + revision: &str, + fetcher: &dyn Fetcher, +) -> Result { + let mut sources: Vec<(String, String)> = Vec::new(); + for e in reg.catalog.rows(id) { + let at = (e.hf_repo.clone(), e.revision.clone()); + if !sources.contains(&at) { + sources.push(at); + } + } + if sources.is_empty() { + sources.push((id.to_string(), revision.to_string())); + } + let mut names = Vec::new(); + for (repo, revision) in &sources { + let files = list_base_files(fetcher, repo, revision)?; + names.extend(files.artifacts.iter().map(|f| f.name.clone())); + } + let Some(file) = names.iter().find(|f| named_variant(f) == variant) else { + bail!( + "no .base in {} is the file of {variant}; there are: {}", + sources + .iter() + .map(|(r, v)| format!("{r}@{v}")) + .collect::>() + .join(", "), + names.join(", ") + ); + }; + if let Some(e) = reg.catalog.rows(id).find(|e| e.file == *file) { + bail!( + "{file} is the catalog's {} of {}: pull {}:{}", + e.quant, + e.id, + e.id, + e.quant + ); + } + Ok(file.clone()) +} + +/// `basert pull `, with no `:variant`. +fn pull_id(reg: &MergedRegistry, args: &PullArgs) -> Result<()> { let root = reg.root.clone(); - let want = quant_token(&args); + let want = quant_token(args); if let Some(file) = &args.file { let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; - return pull_named_file(®, &args, file, &fetcher); + return pull_named_file(reg, args, file, &fetcher); } let r = reg.resolve(&args.id, &args.revision, Some(&want), args.force)?; @@ -714,13 +965,14 @@ pub fn cmd_pull(args: PullArgs) -> Result<()> { // installed_best_variant / the catalog resolver's bit-aware match. let want_bits = base_hub::registry::quant_bits(&want).unwrap_or(&want); if base_hub::registry::quant_bits(variant).unwrap_or(variant) == want_bits { - pull_catalog(&root, &r) + let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; + pull_catalog(&root, &r, &fetcher) } else { let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; let base_files = list_base_files(&fetcher, hf_repo, revision)?; pull_base_direct( &root, - &args, + args, id, hf_repo, revision, @@ -741,11 +993,11 @@ pub fn cmd_pull(args: PullArgs) -> Result<()> { // `.base` repo: it gets the incomplete-publication error, not a // conversion attempt. if base_files.artifacts.is_empty() && base_files.malformed.is_empty() { - pull_and_convert(&root, &args, id, repo, revision) + pull_and_convert(&root, args, id, repo, revision) } else { pull_base_direct( &root, - &args, + args, id, repo, revision, @@ -978,7 +1230,7 @@ fn print_plan(r: &ModelRef, want: &str) { } /// Fast path: download a pre-converted `.base` from the catalog. -fn pull_catalog(root: &Path, r: &ModelRef) -> Result<()> { +fn pull_catalog(root: &Path, r: &ModelRef, fetcher: &dyn Fetcher) -> Result<()> { let ModelRef::Catalog { id, hf_repo, @@ -995,9 +1247,8 @@ fn pull_catalog(root: &Path, r: &ModelRef) -> Result<()> { eprintln!("basert pull v{}", env!("CARGO_PKG_VERSION")); eprintln!(" catalog: {hf_repo}/{file}@{revision} (pre-converted)"); - let fetcher = HfFetcher::new(cache::hf_staging_dir(root))?; // The catalog names the logical `.base`; the repo may hold it as parts. - let artifact = parts::find(&fetcher, hf_repo, revision, file)?; + let artifact = parts::find(fetcher, hf_repo, revision, file)?; let vdir = cache::variant_dir(root, id, variant)?; std::fs::create_dir_all(&vdir)?; @@ -1006,7 +1257,7 @@ fn pull_catalog(root: &Path, r: &ModelRef) -> Result<()> { // Moves the staged download into place (same filesystem), so the pulled // artifact exists exactly once on disk. parts::install( - &fetcher, + fetcher, hf_repo, revision, &artifact, @@ -1020,7 +1271,7 @@ fn pull_catalog(root: &Path, r: &ModelRef) -> Result<()> { std::fs::remove_file(&out).ok(); // The staged bytes are equally bad — no resume value in them, so // clear them and let a retry start from scratch. - fetch::cleanup_staging(&fetcher, hf_repo); + fetch::cleanup_staging(fetcher, hf_repo); bail!("sha256 mismatch for {id}: expected {expected}, got {got_sha}"); } eprintln!(" sha256: verified"); @@ -1038,8 +1289,8 @@ fn pull_catalog(root: &Path, r: &ModelRef) -> Result<()> { Some(got_sha), Some(file.as_str()), )?; - count_download(&fetcher, hf_repo, revision); - fetch::cleanup_staging(&fetcher, hf_repo); + count_download(fetcher, hf_repo, revision); + fetch::cleanup_staging(fetcher, hf_repo); eprintln!("installed {id} [{variant}] → {}", out.display()); Ok(()) } @@ -1178,6 +1429,8 @@ fn pull_and_convert( let fetcher = HfFetcher::new(cache::hf_staging_dir(root))?; let snapshot = download_source(repo, revision, &fetcher)?; + let (quantize_drafter, keep_drafter_fc_f16) = + pull_drafter_flags(drafter_sidecar_arch(&snapshot)); let conv = ConvertArgs { input: snapshot, output: Some(out.clone()), @@ -1190,7 +1443,13 @@ fn pull_and_convert( awq_profile: None, allow_quant_from_quant: false, no_mlx_passthrough: false, - quantize_drafter: false, + // Drafter sidecars (DFlash / DSpark / EAGLE-3) are pulled at the + // target scheme, as the catalog's default-q4 drafters are; which taps + // follow is pull_drafter_flags' call. A target bundle keeps the + // previous behaviour — the flag would also quantize an in-bundle MTP + // head, which has no acceptance measurement. + quantize_drafter, + keep_drafter_fc_f16, speculator_lm_head: false, mtp_draft_ranking: None, mtp_draft_vocab: None, @@ -1283,6 +1542,34 @@ fn choose_profile( Ok((target_str(args.target).to_string(), None, None)) } +/// The drafter sidecar architecture the snapshot's `config.json` names +/// (`dflash`, `dspark` or `eagle3`): the same detection `convert_hf` routes +/// on. None for an ordinary LM, and when the config cannot be read, so an odd +/// snapshot converts exactly as before. +fn drafter_sidecar_arch(snapshot: &Path) -> Option<&'static str> { + let bytes = std::fs::read(snapshot.join("config.json")).ok()?; + let cfg: serde_json::Value = serde_json::from_slice(&bytes).ok()?; + base_arch::dflash::hf_mapper_for_architectures(&cfg) + .or_else(|| base_arch::dflash::hf_mapper_for_eagle3(&cfg)) + .map(base_arch::HfMapper::canonical_arch) +} + +/// `(quantize_drafter, keep_drafter_fc_f16)` for a convert-on-pull of a +/// snapshot whose sidecar arch is `sidecar` (None: a target model). A +/// sidecar's transformer layers follow the target scheme, as the catalog's +/// default-q4 drafters do. Its `fc.{i}` taps follow it for the DFlash family +/// only, where 4-bit taps were measured (acceptance unchanged on +/// Qwen3.8-27B-DFlash2 and Qwen3.6-35B-A3B-DFlash2, 2026-10-19); DSpark and +/// EAGLE-3 taps stay f16 until measured. `base-convert --quantize-drafter` +/// quantizes them regardless. +fn pull_drafter_flags(sidecar: Option<&str>) -> (bool, bool) { + match sidecar { + None => (false, false), + Some("dflash") => (true, false), + Some(_) => (true, true), + } +} + /// Read `model_type` from a fetched `config.json` and fail early when no /// converter supports it — so an unsupported repo errors before its weights /// are downloaded. `text_config.model_type` is consulted as a fallback for @@ -1552,9 +1839,9 @@ pub fn cmd_catalog_scan(org: String, out: Option, dry_run: bool) -> Res } let path = out.unwrap_or_else(|| { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .map(|p| p.join("base-hub/catalog.json")) + std::env::current_exe() + .ok() + .and_then(|exe| workspace_catalog(&exe)) .unwrap_or_else(|| PathBuf::from("catalog.json")) }); if dry_run { @@ -1573,6 +1860,16 @@ pub fn cmd_catalog_scan(org: String, out: Option, dry_run: bool) -> Res Ok(()) } +/// The bundled catalog's source in the workspace a source-built launcher sits +/// in (`/target//basert`): `catalog-scan`'s in-place +/// default. Found at run time because `env!("CARGO_MANIFEST_DIR")` compiled +/// the build machine's checkout path into every release binary. +fn workspace_catalog(exe: &Path) -> Option { + exe.ancestors() + .map(|dir| dir.join("crates/base-hub/catalog.json")) + .find(|catalog| catalog.is_file()) +} + /// Write the manifest a split bundle needs beside its parts. /// /// `bundle` is the logical `.base` path the parts are named after @@ -1637,6 +1934,50 @@ mod tests { use super::*; use base_hub::fetch::MockFetcher; + #[test] + fn a_pulled_snapshot_is_a_drafter_sidecar_by_what_its_config_names() { + let tmp = tempfile::tempdir().unwrap(); + assert_eq!(drafter_sidecar_arch(tmp.path()), None, "no config.json"); + let config = tmp.path().join("config.json"); + for (text, arch) in [ + ( + r#"{"architectures": ["Qwen3ForCausalLM"], "model_type": "qwen3"}"#, + None, + ), + ( + r#"{"architectures": ["DFlashDraftModel"], "model_type": "qwen3"}"#, + Some("dflash"), + ), + ( + r#"{"architectures": ["DFlash2DraftModel"]}"#, + Some("dflash"), + ), + (r#"{"architectures": ["Qwen3DSparkModel"]}"#, Some("dspark")), + ( + r#"{"architectures": ["LlamaForCausalLMEagle3"]}"#, + Some("eagle3"), + ), + // nebius' EAGLE-3 heads say LlamaForCausalLM: one layer with a + // draft vocabulary. + ( + r#"{"architectures": ["LlamaForCausalLM"], "num_hidden_layers": 1, "draft_vocab_size": 32000}"#, + Some("eagle3"), + ), + ("not json", None), + ] { + std::fs::write(&config, text).unwrap(); + assert_eq!(drafter_sidecar_arch(tmp.path()), arch, "{text}"); + } + } + + #[test] + fn a_pull_quantizes_a_sidecars_layers_and_only_the_measured_fc_taps() { + assert_eq!(pull_drafter_flags(None), (false, false)); + assert_eq!(pull_drafter_flags(Some("dflash")), (true, false)); + assert_eq!(pull_drafter_flags(Some("dspark")), (true, true)); + assert_eq!(pull_drafter_flags(Some("eagle3")), (true, true)); + } + #[test] fn bundled_computearena_harness_requires_a_sibling_and_respects_overrides() { let tmp = tempfile::tempdir().unwrap(); @@ -1652,6 +1993,22 @@ mod tests { assert_eq!(bundled_computearena_harness(None, false), None); } + #[test] + fn catalog_scan_writes_the_bundled_catalog_of_a_source_built_launcher() { + let tmp = tempfile::tempdir().unwrap(); + let workspace = tmp.path().join("baseRT/tools/base-convert"); + let catalog = workspace.join("crates/base-hub/catalog.json"); + std::fs::create_dir_all(catalog.parent().unwrap()).unwrap(); + std::fs::create_dir_all(workspace.join("target/release")).unwrap(); + std::fs::write(&catalog, b"{}").unwrap(); + assert_eq!( + workspace_catalog(&workspace.join("target/release/basert")), + Some(catalog) + ); + // An installed launcher has no workspace around it. + assert_eq!(workspace_catalog(&tmp.path().join("bin/basert")), None); + } + #[test] fn bundled_computearena_harness_supports_the_source_build_layout() { let tmp = tempfile::tempdir().unwrap(); @@ -2479,6 +2836,280 @@ mod tests { assert_eq!(std::fs::read(out(&good)).unwrap(), b"medium"); } + /// A Hub that must not be asked anything. + struct Offline; + + impl Fetcher for Offline { + fn get_file(&self, _: &str, _: &str, f: &str) -> Result { + bail!("no download expected ({f})") + } + fn list_files(&self, repo: &str, _: &str) -> Result> { + bail!("no listing expected ({repo})") + } + } + + fn with_variant(id: &str, dry_run: bool) -> PullArgs { + PullArgs { + id: id.into(), + profile: None, + target: TargetScheme::BaseQ4, + revision: "main".into(), + file: None, + force: false, + dry_run, + } + } + + fn pull_with_variant( + reg: &MergedRegistry, + fetcher: &dyn Fetcher, + args: &PullArgs, + ) -> Result<()> { + let (id, variant) = args.id.split_once(':').unwrap(); + pull_variant(reg, args, id, variant, fetcher) + } + + /// `basert pull :` takes the variant as `basert list` names + /// it: an installed one is left as it is without asking the Hub, a + /// catalog one is its row's file (refused for another backend), a + /// `file-…` one is the file it was named for, a quant is a `--target`, + /// and anything else is an error naming the catalog's variants. + #[test] + fn a_pull_names_a_variant_as_basert_list_does() { + let here = CatalogRegistry::client_backend(); + let there = if here == "cuda" { "metal" } else { "cuda" }; + let tmp = tempfile::tempdir().unwrap(); + let theirs = format!("cat-{there}-q4mix.base"); + fixture_repo( + tmp.path(), + "basecompute/cat", + &[ + ("cat-Q4.base", b"q4"), + ("cat-Q8.base", b"q8"), + ("cat-F16.base", b"f16"), + (&theirs, b"theirs"), + ("cat-Medium.base", b"medium"), + ("Extra-Q4.base", b"extra"), + ], + ); + let row = |quant: &str, file: &str, backend: &str| { + format!( + r#"{{"id":"basecompute/cat","hf_repo":"basecompute/cat","quant":"{quant}","file":"{file}"{backend}}}"# + ) + }; + let reg = MergedRegistry::new( + tmp.path().join("cache"), + catalog( + &[ + row("default-q4", "cat-Q4.base", ""), + row("default-q8", "cat-Q8.base", ""), + row("default-f16", "cat-F16.base", ""), + row( + &format!("{there}-q4mix"), + &theirs, + &format!(r#","backend":"{there}""#), + ), + row( + &format!("{there}-q6"), + "cat-Q6.base", + &format!(r#","backend":"{there}""#), + ), + ] + .join(","), + ), + ); + let fetcher = MockFetcher::new(tmp.path()); + let cat = reg.root.join("basecompute/cat"); + let q8 = cat.join("default-q8/model.base"); + + // a catalog variant: its row's file, under its name + let args = with_variant("basecompute/cat:default-q8", false); + pull_with_variant(®, &fetcher, &args).unwrap(); + assert_eq!(std::fs::read(&q8).unwrap(), b"q8"); + assert!( + !cat.join("default-q4").exists(), + "not the catalog's default" + ); + + // installed: left as it is, the Hub unasked, unless --force + std::fs::write(&q8, b"cached").unwrap(); + for dry in [false, true] { + let args = with_variant("basecompute/cat:default-q8", dry); + pull_with_variant(®, &Offline, &args).unwrap(); + } + assert_eq!(std::fs::read(&q8).unwrap(), b"cached"); + let mut force = with_variant("basecompute/cat:default-q8", false); + force.force = true; + pull_with_variant(®, &fetcher, &force).unwrap(); + assert_eq!(std::fs::read(&q8).unwrap(), b"q8"); + + // another backend's variant is refused, and nothing is downloaded + let args = with_variant(&format!("basecompute/cat:{there}-q4mix"), false); + let err = pull_with_variant(®, &fetcher, &args).unwrap_err(); + assert!( + err.to_string().contains(&format!("the {there} bundle")), + "{err}" + ); + assert!(!cat.join(format!("{there}-q4mix")).exists()); + + // a `file-…` variant: the file it was named for + let medium = named_variant("cat-Medium.base"); + let args = with_variant(&format!("basecompute/cat:{medium}"), false); + pull_with_variant(®, &fetcher, &args).unwrap(); + assert_eq!( + std::fs::read(cat.join(&medium).join("model.base")).unwrap(), + b"medium" + ); + let args = with_variant("basecompute/cat:file-cat-large-00000000", true); + let err = pull_with_variant(®, &fetcher, &args).unwrap_err(); + assert!(err.to_string().contains("cat-Medium.base"), "{err}"); + // the catalog's own file is its row's variant, never a `file-…` one + let args = with_variant( + &format!("basecompute/cat:{}", named_variant("cat-Q4.base")), + true, + ); + let err = pull_with_variant(®, &fetcher, &args).unwrap_err(); + assert!( + err.to_string().contains("pull basecompute/cat:default-q4"), + "{err}" + ); + + // a `file-…` variant whose name spells a width is still its file + let extra = named_variant("Extra-Q4.base"); + assert_eq!( + variant_source(®, "basecompute/cat", &extra, "main", &fetcher).unwrap(), + VariantSource::File("Extra-Q4.base".into()) + ); + + // a width named on its own is the catalog's row of it for this + // client, f16 included; one published only for another backend is + // refused + let row_of = + |v: &str| match variant_source(®, "basecompute/cat", v, "main", &Offline).unwrap() { + VariantSource::Catalog(ModelRef::Catalog { variant, file, .. }) => (variant, file), + other => panic!("{v}: {other:?}"), + }; + for (v, row) in [ + ("q4", "default-q4"), + ("q8", "default-q8"), + ("f16", "default-f16"), + ] { + let file = format!("cat-{}.base", v.to_ascii_uppercase()); + assert_eq!(row_of(v), (row.to_string(), file)); + } + let err = variant_source(®, "basecompute/cat", "q6", "main", &Offline) + .unwrap_err() + .to_string(); + assert!(err.contains(&format!("the {here} backend")), "{err}"); + + // a quant is a --target, as --target spells it + for (v, t) in [ + ("q8", TargetScheme::BaseQ8), + ("default-q6", TargetScheme::BaseQ6), + ("base_q2", TargetScheme::BaseQ2), + ("base-q3", TargetScheme::BaseQ3), + ("bf16", TargetScheme::Bf16), + ] { + assert_eq!( + variant_source(®, "org/model", v, "main", &Offline).unwrap(), + VariantSource::Quant(t) + ); + } + // anything else names the variants this client can pull, a name + // that merely contains or ends in a width included + for v in [ + "large", + "default-q8-custom", + "not-a-variant-f16", + "typo-q8", + "q8-custom", + ] { + let err = variant_source(®, "basecompute/cat", v, "main", &Offline) + .unwrap_err() + .to_string(); + assert!( + err.contains("the catalog lists default-q4, default-q8, default-f16") + && !err.contains(there), + "{v}: {err}" + ); + let err = variant_source(®, "org/model", v, "main", &Offline).unwrap_err(); + assert!(err.to_string().contains("has no variant"), "{v}: {err}"); + } + + // a flag that names the variant too is refused, as is an empty one + let mut file = with_variant("basecompute/cat:default-q8", true); + file.file = Some("cat-Q8.base".into()); + let mut target = with_variant("basecompute/cat:default-q8", true); + target.target = TargetScheme::BaseQ8; + let mut profile = with_variant("basecompute/cat:default-q8", true); + profile.profile = Some("default-q8.json".into()); + for (args, flag) in [ + (file, "--file"), + (target, "--target"), + (profile, "--profile"), + ] { + let err = pull_with_variant(®, &Offline, &args).unwrap_err(); + assert!(err.to_string().contains(flag), "{err}"); + } + let args = with_variant("basecompute/cat:", true); + let err = pull_with_variant(®, &Offline, &args).unwrap_err(); + assert!(err.to_string().contains("empty variant"), "{err}"); + } + + /// The row a variant names is the one pulled, from its own repo, even + /// when another row of the id publishes a file of the same path: by its + /// name, by its width, and by a re-pull. + #[test] + fn a_variant_pulls_its_row_when_rows_share_a_file_name() { + let tmp = tempfile::tempdir().unwrap(); + fixture_repo(tmp.path(), "basecompute/cat-a", &[("model.base", b"a")]); + fixture_repo(tmp.path(), "basecompute/cat-b", &[("model.base", b"b")]); + let row = |repo: &str, quant: &str| { + format!( + r#"{{"id":"basecompute/cat","hf_repo":"basecompute/{repo}","quant":"{quant}","file":"model.base"}}"# + ) + }; + let reg = MergedRegistry::new( + tmp.path().join("cache"), + catalog(&[row("cat-a", "default-q4"), row("cat-b", "default-q8")].join(",")), + ); + let fetcher = MockFetcher::new(tmp.path()); + let installed = |variant: &str| { + std::fs::read( + reg.root + .join("basecompute/cat") + .join(variant) + .join("model.base"), + ) + .unwrap() + }; + + let args = with_variant("basecompute/cat:default-q8", false); + pull_with_variant(®, &fetcher, &args).unwrap(); + assert_eq!(installed("default-q8"), b"b"); + let args = with_variant("basecompute/cat:q4", false); + pull_with_variant(®, &fetcher, &args).unwrap(); + assert_eq!(installed("default-q4"), b"a"); + assert_eq!( + held(®, "basecompute/cat", "default-q8").as_deref(), + Some("model.base") + ); + + let mut force = with_variant("basecompute/cat:default-q8", false); + force.force = true; + std::fs::write( + reg.root.join("basecompute/cat/default-q8/model.base"), + b"stale", + ) + .unwrap(); + pull_with_variant(®, &fetcher, &force).unwrap(); + assert_eq!(installed("default-q8"), b"b"); + for dry in [false, true] { + let args = with_variant("basecompute/cat:q8", dry); + pull_with_variant(®, &Offline, &args).unwrap(); + } + } + /// Fetcher owning an hf-hub-style staging tree (blob + snapshot symlink), /// mirroring what `HfFetcher` leaves on disk mid-pull. struct StagedFetcher { diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index bd61773..ed07d37 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -158,14 +158,24 @@ struct ConvertArgs { #[arg(long)] no_mlx_passthrough: bool, /// Block-drafter / EAGLE-3 sidecars only: quantize the drafter's own - /// transformer layers (`layers.*` q/k/v/o and FFN projections) with - /// `--target` instead of keeping the whole sidecar f16. The heads that - /// define the scheme (fc taps, hidden_norm, Markov / confidence heads, - /// the EAGLE-3 lm_head and d2t) and every norm stay f16. A DFlash-b16 - /// head for Qwen3-4B is 1 GB f16 — streamed on every block step; q8 - /// halves it, q4 quarters it, at a small cost in acceptance. + /// transformer layers (`layers.*` q/k/v/o and FFN projections) and the + /// block drafter's per-tap context projection `fc.{i}` with `--target` + /// instead of keeping the whole sidecar f16. The remaining heads that + /// define the scheme (hidden_norm, Markov / confidence heads, DFlash 2's + /// selector codebooks, the EAGLE-3 lm_head and d2t) and every norm stay + /// f16. A DFlash-b16 head for Qwen3-4B is 1 GB f16 — streamed on every + /// block step; q8 halves it, q4 quarters it, at a small cost in + /// acceptance. The fc taps are streamed on every round's context pass + /// (262 MB f16 on Qwen3.8-27B-DFlash2, 67 MB on Qwen3.6-35B-A3B-DFlash2); + /// measured 2026-10-19 at 4-bit: acceptance unchanged on both, 0.9 / 0.2 ms + /// less per round. `basert pull` converts a drafter sidecar this way, the + /// fc taps included for the DFlash family only, where that was measured. #[arg(long)] quantize_drafter: bool, + /// With --quantize-drafter: keep the block drafter's `fc.{i}` taps in f16 + /// (the pre-2026-10-19 bundles). The 1/256 fold applies either way. + #[arg(long, requires = "quantize_drafter")] + keep_drafter_fc_f16: bool, /// GGUF sources only: copy Q3_K / Q4_K / Q5_K / Q6_K / IQ4_XS and IQ2_XS / /// IQ2_S / IQ3_XXS / IQ3_S super-blocks into the bundle VERBATIM instead /// of dequantizing and re-packing them. @@ -371,7 +381,9 @@ enum AwqMode { #[derive(Parser, Debug)] struct PullArgs { /// Model id: `basecompute/` (pre-converted, from the catalog) or - /// `org/model` (a raw HF repo, downloaded + converted locally). + /// `org/model` (a raw HF repo, downloaded + converted locally). Add + /// `:` for that variant, as `basert list` names it + /// (`default-q8`, `cuda-q4mix`, `file-…`), or a quant (`q8`). id: String, /// Override the auto-selected quant profile (path to a profile JSON). #[arg(long)] @@ -402,7 +414,8 @@ struct CatalogScanArgs { /// Organization to scan. #[arg(long, default_value = "basecompute")] org: String, - /// Where to write the catalog. Defaults to the bundled one, in place. + /// Where to write the catalog. Defaults to the bundled one, in place, when + /// this launcher was built from source; to catalog.json here otherwise. #[arg(long)] out: Option, /// Report what would change without writing anything. @@ -3701,6 +3714,69 @@ fn dequant_to_f16(source_format: &str, src_quantized: bool, canonical: &str) -> src_quantized && source_format == "mlx_safetensors" && f16_consumer(canonical) } +/// Whether a drafter sidecar's tensor keeps f16: every one of them does +/// without `--quantize-drafter`; with it, the transformer layers (`layers.*`) +/// and, unless `keep_fc_f16`, the `fc.{i}` context taps follow the target +/// scheme. DFlash 2's conv `base_kernel` ([2, K, D]) is read as raw halves, +/// not a projection, so it stays f16 under either rule (its +/// `kernel_projection` quantizes). The norms, the selector codebooks, the +/// Markov / confidence heads and EAGLE-3's lm_head and d2t define the scheme +/// and stay f16. 1-D tensors and biases stay raw anyway (`must_stay_raw`). +fn drafter_tensor_stays_f16(quantize_drafter: bool, keep_fc_f16: bool, canonical: &str) -> bool { + let follows_target = + canonical.starts_with("layers.") || (!keep_fc_f16 && canonical.starts_with("fc.")); + !(quantize_drafter && follows_target && !canonical.ends_with(".base_kernel")) +} + +#[cfg(test)] +mod drafter_policy_tests { + use super::drafter_tensor_stays_f16; + + #[test] + fn without_the_flag_every_sidecar_tensor_stays_f16() { + for name in [ + "layers.0.self_attn.q_proj.weight", + "fc.0.weight", + "hidden_norm.weight", + "lm_head.weight", + ] { + assert!(drafter_tensor_stays_f16(false, false, name), "{name}"); + } + } + + #[test] + fn the_flag_quantizes_the_layers_and_the_fc_taps_and_nothing_else() { + for name in [ + "layers.0.self_attn.q_proj.weight", + "layers.4.mlp.down_proj.weight", + "fc.0.weight", + "fc.7.weight", + ] { + assert!(!drafter_tensor_stays_f16(true, false, name), "{name}"); + } + for name in [ + "hidden_norm.weight", + "markov_head.weight", + "confidence_head.weight", + "lm_head.weight", + "d2t", + "layers.0.conv.base_kernel", + ] { + assert!(drafter_tensor_stays_f16(true, false, name), "{name}"); + } + } + + #[test] + fn keeping_the_fc_taps_f16_leaves_the_layers_quantized() { + assert!(drafter_tensor_stays_f16(true, true, "fc.0.weight")); + assert!(!drafter_tensor_stays_f16( + true, + true, + "layers.0.mlp.up_proj.weight" + )); + } +} + #[allow(clippy::too_many_arguments)] fn convert_generic( input: &std::path::Path, @@ -4134,20 +4210,14 @@ fn convert_generic( .cloned() .chain(speculator_mapped.iter().cloned()) .collect(); - // Block-drafter sidecars stay f16 end to end (acceptance rate is what - // the scheme buys; the block step is small next to the target's verify). + // Drafter sidecars stay f16 (acceptance rate is what the scheme buys; the + // block step is small next to the target's verify) except for what + // --quantize-drafter names: see drafter_tensor_stays_f16. let is_drafter_arch = base_arch::dflash::is_block_drafter_arch(canonical_arch) || base_arch::dflash::is_eagle3_arch(canonical_arch); for (src_name, canonical) in &mapped { - // --quantize-drafter: the drafter's transformer layers follow - // --target; everything else in the sidecar (fc taps, norms, heads, - // d2t) stays f16. - // DFlash 2's conv `base_kernel` ([2, K, D]) is read as raw halves, - // not a projection — it stays f16 (its kernel_projection quantizes). let drafter_f16 = is_drafter_arch - && !(ctx.quantize_drafter - && canonical.starts_with("layers.") - && !canonical.ends_with(".base_kernel")); + && drafter_tensor_stays_f16(ctx.quantize_drafter, ctx.keep_drafter_fc_f16, canonical); let is_spec = spec_names.contains(canonical); // --quantize-drafter also quantizes an in-bundle speculator's transformer // layer (`mtp.layers.*`): a Qwen3.8-27B MTP head is ~700 MB f16 and @@ -6341,6 +6411,8 @@ struct QuantContext { q4_passthrough: bool, /// --quantize-drafter (sidecars): drafter `layers.*` follow the target scheme. quantize_drafter: bool, + /// --keep-drafter-fc-f16: the drafter's `fc.{i}` taps stay f16 under it. + keep_drafter_fc_f16: bool, /// `--speculator-lm-head`: the MTP head gets its own `mtp.lm_head.weight`. speculator_lm_head: bool, /// `--mtp-draft-ranking`: ranked token ids for the MTP head's drafting @@ -6418,6 +6490,7 @@ impl QuantContext { allow_quant_from_quant: args.allow_quant_from_quant, q4_passthrough: !args.no_mlx_passthrough, quantize_drafter: args.quantize_drafter, + keep_drafter_fc_f16: args.keep_drafter_fc_f16, speculator_lm_head: args.speculator_lm_head, mtp_draft_ranking, mtp_draft_vocab: args.mtp_draft_vocab, diff --git a/base-convert/crates/base-convert/src/superfluid.rs b/base-convert/crates/base-convert/src/superfluid.rs index e3ca0ad..264dc76 100644 --- a/base-convert/crates/base-convert/src/superfluid.rs +++ b/base-convert/crates/base-convert/src/superfluid.rs @@ -1136,15 +1136,17 @@ mod tests { fn the_engine_abi_is_the_one_the_header_states() { // The headers are at the repository's root: four directories up here, // three in the public repository, where this crate is `base-convert/`. + // The crate from crates.io carries its own copy. let crate_dir = Path::new(env!("CARGO_MANIFEST_DIR")); let header = [ "../../../../include/baseRT/baseRT.h", "../../../include/baseRT/baseRT.h", + "include/baseRT/baseRT.h", ] .iter() .map(|rel| crate_dir.join(rel)) .find(|p| p.is_file()) - .expect("include/baseRT/baseRT.h at the repository's root"); + .expect("include/baseRT/baseRT.h at the repository's root, or in the crate"); let text = fs::read_to_string(&header).unwrap(); let stated = text .lines() diff --git a/base-convert/crates/base-convert/tests/whisper_quant.rs b/base-convert/crates/base-convert/tests/whisper_quant.rs index 3c9e736..0b24faa 100644 --- a/base-convert/crates/base-convert/tests/whisper_quant.rs +++ b/base-convert/crates/base-convert/tests/whisper_quant.rs @@ -19,8 +19,14 @@ fn bin() -> &'static str { } fn profiles_dir() -> PathBuf { + // The copy packaged in the crate (crates.io), else the workspace's: // crates/base-convert → ../../profiles - Path::new(env!("CARGO_MANIFEST_DIR")) + let crate_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let packaged = crate_dir.join("profiles"); + if packaged.is_dir() { + return packaged; + } + crate_dir .parent() .unwrap() .parent() diff --git a/base-convert/crates/base-hub/src/registry.rs b/base-convert/crates/base-hub/src/registry.rs index 5e8ad48..db31ebf 100644 --- a/base-convert/crates/base-hub/src/registry.rs +++ b/base-convert/crates/base-hub/src/registry.rs @@ -273,7 +273,8 @@ impl CatalogRegistry { .unwrap_or(true) } - fn entry_to_ref(e: &crate::catalog::CatalogEntry) -> ModelRef { + /// The [`ModelRef`] a catalog row resolves to. + pub fn entry_to_ref(e: &crate::catalog::CatalogEntry) -> ModelRef { ModelRef::Catalog { id: e.id.clone(), hf_repo: e.hf_repo.clone(), diff --git a/base-convert/crates/base-quant/src/profile.rs b/base-convert/crates/base-quant/src/profile.rs index c1287b4..39afcac 100644 --- a/base-convert/crates/base-quant/src/profile.rs +++ b/base-convert/crates/base-quant/src/profile.rs @@ -432,32 +432,24 @@ mod tests { assert!(format!("{err}").contains("e4m3")); } + /// A shipped profile: the copy packaged in the crate (crates.io), else + /// the workspace's `profiles/`, beside `crates/`. + fn shipped_profile(name: &str) -> std::path::PathBuf { + let crate_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let packaged = crate_dir.join("profiles").join(name); + if packaged.is_file() { + packaged + } else { + crate_dir.join("../../profiles").join(name) + } + } + /// The shipped default profiles must validate. This catches typos /// in on-disk profile JSON before they hit the converter. #[test] fn shipped_default_profiles_parse() { - // Profiles live at /tools/base-convert/profiles/. Walk up - // from CARGO_MANIFEST_DIR (= the base-quant crate) to that path. - let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let profiles_dir = std::path::Path::new(&manifest) - .parent() // crates/ - .unwrap() - .parent() // base-convert/ - .unwrap() - .join("profiles"); - let names = [ - "default-q4.json", - "default-q8.json", - "dense-q4mix.json", - "moe-q4mix.json", - "q3-aggressive.json", - "gemma4-moe-q4mix.json", - "gemma4-moe-q4all.json", - "gemma4-moe-mlx.json", - ]; - for name in names { - let path = profiles_dir.join(name); - let p = QuantProfile::from_path(&path) + for name in ["default-q4.json", "default-q8.json"] { + let p = QuantProfile::from_path(&shipped_profile(name)) .unwrap_or_else(|e| panic!("profile {name} failed: {e}")); // Sanity: the catch-all "**.weight" rule covers a generic // tensor name, so resolve never returns None for plausible @@ -468,216 +460,6 @@ mod tests { } } - /// MoE profile must route experts to q4 and shared_ffn / lm_head - /// to q8 — the canonical mixed-precision layout. - #[test] - fn moe_q4mix_profile_routes_correctly() { - let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let path = std::path::Path::new(&manifest) - .parent() - .unwrap() - .parent() - .unwrap() - .join("profiles") - .join("moe-q4mix.json"); - let p = QuantProfile::from_path(&path).unwrap(); - - // MoE expert: q4 / gs=64. - let expert = p - .resolve("model.layers.0.mlp.experts.0.gate_proj.weight") - .unwrap(); - assert_eq!(expert.dtype, TensorDtype::BaseQ4); - assert_eq!(expert.group_size, 64); - - // Shared FFN (Qwen3-MoE / DeepSeek style): q8 / gs=128. - let shared = p - .resolve("model.layers.0.mlp.shared_experts.gate_proj.weight") - .unwrap(); - assert_eq!(shared.dtype, TensorDtype::BaseQ8); - assert_eq!(shared.group_size, 128); - - // lm_head: q8. - let lm = p.resolve("lm_head.weight").unwrap(); - assert_eq!(lm.dtype, TensorDtype::BaseQ8); - - // Router stays in fp (kernel reads f16; profile uses f16 - // since the runtime's norm/router kernels assume half). - let router = p.resolve("model.layers.0.mlp.router.weight").unwrap(); - assert!(matches!(router.dtype, TensorDtype::F16 | TensorDtype::Bf16)); - } - - /// Gemma-4 q4mix keeps the attention key projection at q8 (q / v / o - /// stay q4). Gemma-4 attends with scale 1.0 over RMS-normed q and k, - /// and every later query reads a position's cached key, so 4-bit keys - /// blunt the heads that point back at an earlier occurrence of a token: - /// the 26B-A4B garbles identifiers it is copying (`y_position_m` in a - /// script it is writing). 8-bit keys alone restore it. - #[test] - fn gemma4_moe_q4mix_keeps_k_proj_q8() { - let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let path = std::path::Path::new(&manifest) - .parent() - .unwrap() - .parent() - .unwrap() - .join("profiles") - .join("gemma4-moe-q4mix.json"); - let p = QuantProfile::from_path(&path).unwrap(); - for name in [ - "model.language_model.layers.0.self_attn.k_proj.weight", - "layers.5.self_attn.k_proj.weight", - "blk.3.attn_k.weight", - ] { - let r = p.resolve(name).unwrap(); - assert_eq!(r.dtype, TensorDtype::BaseQ8, "{name}"); - assert_eq!(r.group_size, 64, "{name}"); - assert_eq!(r.scale_dtype, ScaleDtype::Bf16, "{name}"); - } - for name in [ - "model.language_model.layers.0.self_attn.q_proj.weight", - "model.language_model.layers.0.self_attn.v_proj.weight", - "model.language_model.layers.0.self_attn.o_proj.weight", - "model.vision_tower.encoder.layers.0.self_attn.k_proj.linear.weight", - ] { - let r = p.resolve(name).unwrap(); - assert_eq!(r.dtype, TensorDtype::BaseQ4, "{name}"); - assert_eq!(r.group_size, 64, "{name}"); - } - } - - /// q3-aggressive profile routes MLP / experts to q3 / gs=32 per spec. - #[test] - fn q3_aggressive_profile_uses_q3_gs32() { - let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let path = std::path::Path::new(&manifest) - .parent() - .unwrap() - .parent() - .unwrap() - .join("profiles") - .join("q3-aggressive.json"); - let p = QuantProfile::from_path(&path).unwrap(); - - let mlp = p.resolve("model.layers.5.mlp.gate_proj.weight").unwrap(); - assert_eq!(mlp.dtype, TensorDtype::BaseQ3); - assert_eq!(mlp.group_size, 32); - - // Attention stays at q4 even in this profile. - let attn = p.resolve("model.layers.5.self_attn.q_proj.weight").unwrap(); - assert_eq!(attn.dtype, TensorDtype::BaseQ4); - assert_eq!(attn.group_size, 64); - } - - /// `gemma4-moe-mlx` mirrors `mlx-community/gemma-4-26b-a4b-it-4bit`'s - /// per-tensor `quantization` block: q4 default, q8 on the shared-dense - /// FFN (`mlp.{gate,up,down}_proj`) and the router projection. Embed - /// drops to q4 to match MLX (lm_head is tied; the canonical name - /// remap routes through `embed_tokens.weight`). This test pins the - /// routing so a future profile edit can't silently drift it. - #[test] - fn gemma4_moe_mlx_profile_matches_mlx_quant_block() { - let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let path = std::path::Path::new(&manifest) - .parent() - .unwrap() - .parent() - .unwrap() - .join("profiles") - .join("gemma4-moe-mlx.json"); - let p = QuantProfile::from_path(&path).unwrap(); - - // Shared dense FFN — q8/gs=64 (only path that sees every token - // on top of the routed experts; MLX upgrades to 8 bit so q4 - // residual-stream noise doesn't compound through 30 layers). - for proj in ["gate_proj", "up_proj", "down_proj"] { - let name = format!("model.layers.0.mlp.{proj}.weight"); - let r = p - .resolve(&name) - .unwrap_or_else(|| panic!("no rule for {name}")); - assert_eq!(r.dtype, TensorDtype::BaseQ8, "mlp.{proj} should be q8"); - assert_eq!(r.group_size, 64, "mlp.{proj} should be gs=64"); - } - - // Router projection — q8/gs=64. Picks expert assignments; - // q4 noise here silently re-routes to wrong experts which is - // unrecoverable, while the bytes saved are negligible. - for name in [ - "model.language_model.layers.0.router.proj.weight", - "model.layers.0.mlp.router.weight", - "model.layers.0.ffn_gate_inp.weight", - ] { - let r = p - .resolve(name) - .unwrap_or_else(|| panic!("no rule for {name}")); - assert_eq!(r.dtype, TensorDtype::BaseQ8, "{name} should be q8"); - assert_eq!(r.group_size, 64, "{name} should be gs=64"); - } - - // Routed experts — q4/gs=64 (the bulk of model bytes; q4 noise - // averages across the 8/128 experts that fire per token). - for name in [ - "model.layers.0.experts.gate_up_proj", - "model.layers.0.ffn_gate_up_exps.weight", - "model.layers.0.ffn_down_exps.weight", - ] { - let r = p - .resolve(name) - .unwrap_or_else(|| panic!("no rule for {name}")); - assert_eq!(r.dtype, TensorDtype::BaseQ4, "{name} should be q4"); - assert_eq!(r.group_size, 64, "{name} should be gs=64"); - } - - // Attention projections — q4/gs=64. - for proj in ["q_proj", "k_proj", "v_proj", "o_proj"] { - let name = format!("model.layers.0.self_attn.{proj}.weight"); - let r = p - .resolve(&name) - .unwrap_or_else(|| panic!("no rule for {name}")); - assert_eq!( - r.dtype, - TensorDtype::BaseQ4, - "self_attn.{proj} should be q4" - ); - } - - // Embed_tokens — q4/gs=64 to match MLX (also drops lm_head with - // tied embeddings). Saves ~1 GB on 26B-A4B's 262144 vocab vs f16. - let embed = p.resolve("model.embed_tokens.weight").unwrap(); - assert_eq!(embed.dtype, TensorDtype::BaseQ4); - assert_eq!(embed.group_size, 64); - - // Norms stay f16 — the runtime's rmsnorm_f16 kernel reads weight - // buffers as `half *`, and bf16 norms break that read. - for name in [ - "model.layers.0.input_layernorm.weight", - "model.layers.0.post_feedforward_layernorm.weight", - "model.layers.0.post_feedforward_layernorm_2.weight", - "model.layers.0.post_attention_layernorm.weight", - "model.layers.0.pre_feedforward_layernorm.weight", - "model.layers.0.pre_feedforward_layernorm_2.weight", - ] { - let r = p - .resolve(name) - .unwrap_or_else(|| panic!("no rule for {name}")); - assert_eq!(r.dtype, TensorDtype::F16, "{name} should be f16"); - } - - // Per-layer / per-expert scalar weights stay f16 (small, sensitive). - for name in [ - "model.layers.0.layer_scalar", - "model.layers.0.layer_out_scale.weight", - "model.layers.0.router.per_expert_scale", - "model.layers.0.ffn_down_exps.scale", - "model.layers.0.router.scale", - "model.layers.0.ffn_gate_inp.scale", - ] { - let r = p - .resolve(name) - .unwrap_or_else(|| panic!("no rule for {name}")); - assert_eq!(r.dtype, TensorDtype::F16, "{name} should be f16"); - } - } - /// The whisper profiles quantize ONLY the block linear projections /// (attn/cross_attn query/key/value/out, mlp.0/mlp.2) and keep /// conv, positional embeddings, token_embedding, norms and biases @@ -685,18 +467,11 @@ mod tests { /// packed scheme the engine's quant whisper kernels expect. #[test] fn whisper_profiles_route_correctly() { - let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let profiles_dir = std::path::Path::new(&manifest) - .parent() - .unwrap() - .parent() - .unwrap() - .join("profiles"); for (file, dtype, gs) in [ ("whisper-q8.json", TensorDtype::BaseQ8, 128), ("whisper-q4.json", TensorDtype::BaseQ4, 64), ] { - let p = QuantProfile::from_path(&profiles_dir.join(file)).unwrap(); + let p = QuantProfile::from_path(&shipped_profile(file)).unwrap(); assert_eq!(p.arch, "whisper", "{file}"); // Block linears → quantized. diff --git a/bindings/node/README.md b/bindings/node/README.md index 981a97a..e413dd0 100644 --- a/bindings/node/README.md +++ b/bindings/node/README.md @@ -30,8 +30,12 @@ npm test # struct-ABI + helper tests; no engine/model needed The bindings resolve `libbaseRT.dylib` in this order: 1. `BASERT_LIB_PATH` environment variable (full path to the `.dylib`) -2. `../../build/libbaseRT.dylib` relative to the package (when inside the repo) -3. `build/libbaseRT.dylib` / `libbaseRT.dylib` under the current directory +2. The engine package npm installed for this platform + (`@basecompute/basert-darwin-arm64` / `@basecompute/basert-linux-arm64-cuda`) +3. `../../build/libbaseRT.dylib` relative to the package (when inside the repo) +4. A prebuilt engine from `install.sh` (`$BASERT_INSTALL_DIR`, default + `~/.basert`) or Homebrew (`$(brew --prefix)/opt/basert/libexec`) +5. `build/libbaseRT.dylib` / `libbaseRT.dylib` under the current directory ## Note on threading diff --git a/bindings/node/package.json b/bindings/node/package.json index b1967b1..6e75d94 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@baseRT/node", - "version": "0.4.1", + "version": "0.4.2", "private": true, "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)", "main": "dist/index.js", diff --git a/bindings/node/src/index.ts b/bindings/node/src/index.ts index 6464a96..7b0a7d1 100644 --- a/bindings/node/src/index.ts +++ b/bindings/node/src/index.ts @@ -19,6 +19,7 @@ import koffi from "koffi"; import * as fs from "fs"; +import * as os from "os"; import * as path from "path"; // --------------------------------------------------------------------------- @@ -40,15 +41,43 @@ function libNames(): string[] { } } +// The prebuilt engine is published to npm as one package per platform, listed +// as optionalDependencies of @basecompute/basert; npm installs only the one +// whose os/cpu match this machine. +const ENGINE_PACKAGES: Record = { + "darwin-arm64": "@basecompute/basert-darwin-arm64", + "linux-arm64": "@basecompute/basert-linux-arm64-cuda", +}; + +function enginePackageDir(): string | undefined { + const pkg = ENGINE_PACKAGES[`${process.platform}-${process.arch}`]; + if (!pkg) return undefined; + try { + return path.join(path.dirname(require.resolve(`${pkg}/package.json`)), "engine"); + } catch { + return undefined; // not installed: a source checkout, or --no-optional + } +} + function resolveLibPath(): string { if (process.env.BASERT_LIB_PATH) return process.env.BASERT_LIB_PATH; const here = __dirname; // dist/ or src/ at runtime + // A source checkout's build directory comes before the installed engines + // (install.sh / Homebrew) so a fresh local build is never shadowed by an + // older release. The working directory is tried last: whatever folder the + // script runs in must never outrank an installed engine, or a project could + // ship its own library. + const installDir = process.env.BASERT_INSTALL_DIR || path.join(os.homedir(), ".basert"); const dirs = [ + enginePackageDir(), // npm install @basecompute/basert path.resolve(here, "..", "..", "..", "build"), // bindings/node/{dist,src} -> repo/build path.resolve(here, "..", "..", "build"), + installDir, // install.sh + "/opt/homebrew/opt/basert/libexec", // brew, macOS + "/home/linuxbrew/.linuxbrew/opt/basert/libexec", // brew, Linux path.resolve(process.cwd(), "build"), path.resolve(process.cwd()), - ]; + ].filter((d): d is string => d !== undefined); const names = libNames(); const tried: string[] = []; for (const d of dirs) { @@ -59,7 +88,7 @@ function resolveLibPath(): string { } } throw new Error( - `Cannot find ${names.join(" / ")}. Set BASERT_LIB_PATH or build with \`make shared\`. Looked in:\n ${tried.join( + `Cannot find ${names.join(" / ")}. Install the engine (npm install @basecompute/basert, brew install basecompute/tap/basert, or install.sh), set BASERT_LIB_PATH, or build with \`make shared\`. Looked in:\n ${tried.join( "\n " )}` ); diff --git a/bindings/python/README.md b/bindings/python/README.md index 21a1fc7..1c4e957 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -26,8 +26,11 @@ pip install -e . The bindings look for `libbaseRT.dylib` in this order: 1. `BASERT_LIB_PATH` environment variable (full path to the `.dylib`). -2. `../../build/libbaseRT.dylib` relative to the package source. -3. `build/libbaseRT.dylib` in the current working directory. +2. `_engine/` inside the installed package (the PyPI wheel bundles the engine). +3. `../../build/libbaseRT.dylib` relative to the package source. +4. A prebuilt engine from `install.sh` (`$BASERT_INSTALL_DIR`, default + `~/.basert`) or Homebrew (`$(brew --prefix)/opt/basert/libexec`). +5. `build/libbaseRT.dylib` in the current working directory. ## Usage diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index f919f27..92fb38e 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union -__version__ = "0.4.1" +__version__ = "0.4.2" # --------------------------------------------------------------------------- # Library loading @@ -41,18 +41,35 @@ def _lib_names() -> list[str]: return ["libbaseRT.so"] # Linux / other Unix +def _installed_engine_dirs() -> list[Path]: + """Where install.sh and Homebrew put the prebuilt engine.""" + install_dir = os.environ.get("BASERT_INSTALL_DIR") or str(Path.home() / ".basert") + return [ + Path(install_dir), # install.sh + Path("/opt/homebrew/opt/basert/libexec"), # brew, macOS + Path("/home/linuxbrew/.linuxbrew/opt/basert/libexec"), # brew, Linux + ] + + def _find_library() -> str: """Locate the BaseRT shared library.""" env = os.environ.get("BASERT_LIB_PATH") if env: return env - # Walk upward from this file to find the project build directory, trying - # each platform-appropriate filename in each candidate directory. + # Try each platform-appropriate filename in each candidate directory, in + # order: the engine bundled inside the PyPI wheel, a source checkout's + # build directory, then a prebuilt engine installed by install.sh or + # Homebrew. The checkout comes before the installed copies so a fresh + # local build is never shadowed by an older release. The working + # directory is tried last: whatever folder the script runs in must never + # outrank an installed engine, or a project could ship its own library. here = Path(__file__).resolve().parent search_dirs = [ + here / "_engine", # pip install basert here / ".." / ".." / ".." / "build", # bindings/python/baseRT -> build/ here / ".." / ".." / "build", + *_installed_engine_dirs(), Path("build"), ] names = _lib_names() @@ -63,8 +80,9 @@ def _find_library() -> str: return str(resolved) raise OSError( - f"Cannot find {' / '.join(names)}. Set BASERT_LIB_PATH or build with " - "`make shared`." + f"Cannot find {' / '.join(names)}. Install the engine (`pip install " + "basert`, `brew install basecompute/tap/basert`, or install.sh), set BASERT_LIB_PATH, " + "or build with `make shared`." ) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index b4e8f53..e270336 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -6,7 +6,7 @@ setup( name="baseRT", - version="0.4.1", + version="0.4.2", description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)", long_description=long_description, long_description_content_type="text/markdown", diff --git a/bindings/python/tests/test_baseRT.py b/bindings/python/tests/test_baseRT.py index 564027d..ec25b65 100644 --- a/bindings/python/tests/test_baseRT.py +++ b/bindings/python/tests/test_baseRT.py @@ -452,12 +452,52 @@ def test_raises_when_not_found(self, tmp_path, monkeypatch): # Clear every resolver input. In particular, isolate cwd so a shared # library produced by an earlier test/CI build cannot satisfy the # fallback lookup and make this test order-dependent. + # Also hide any engine installed on this machine (~/.basert, Homebrew). monkeypatch.chdir(tmp_path) with mock.patch.dict(os.environ, {}, clear=True): - with mock.patch("baseRT.__file__", "/nonexistent/bindings/python/baseRT/__init__.py"): + with mock.patch("baseRT.__file__", "/nonexistent/bindings/python/baseRT/__init__.py"), \ + mock.patch("baseRT._installed_engine_dirs", return_value=[]): with pytest.raises(OSError, match="Cannot find"): _find_library() + def test_finds_library_bundled_in_wheel(self, tmp_path): + # pip install basert: site-packages/baseRT/_engine/libbaseRT.* + package_dir = tmp_path / "baseRT" + (package_dir / "_engine").mkdir(parents=True) + fake_lib = package_dir / "_engine" / _LIB_NAME + fake_lib.touch() + fake_init = package_dir / "__init__.py" + fake_init.touch() + with mock.patch.dict(os.environ, {}, clear=True): + with mock.patch("baseRT.__file__", str(fake_init)): + assert _find_library() == str(fake_lib.resolve()) + + def test_finds_library_in_install_dir(self, tmp_path, monkeypatch): + # install.sh: $BASERT_INSTALL_DIR (default ~/.basert) + monkeypatch.chdir(tmp_path) + install_dir = tmp_path / "basert-install" + install_dir.mkdir() + fake_lib = install_dir / _LIB_NAME + fake_lib.touch() + with mock.patch.dict(os.environ, {"BASERT_INSTALL_DIR": str(install_dir)}, clear=True): + with mock.patch("baseRT.__file__", "/nonexistent/bindings/python/baseRT/__init__.py"): + assert _find_library() == str(fake_lib.resolve()) + + def test_install_dir_outranks_working_directory(self, tmp_path, monkeypatch): + # A library in the folder the script runs from must not shadow an + # installed engine, or any project could ship its own libbaseRT. + project = tmp_path / "project" + (project / "build").mkdir(parents=True) + (project / "build" / _LIB_NAME).touch() + monkeypatch.chdir(project) + install_dir = tmp_path / "basert-install" + install_dir.mkdir() + installed_lib = install_dir / _LIB_NAME + installed_lib.touch() + with mock.patch.dict(os.environ, {"BASERT_INSTALL_DIR": str(install_dir)}, clear=True): + with mock.patch("baseRT.__file__", "/nonexistent/bindings/python/baseRT/__init__.py"): + assert _find_library() == str(installed_lib.resolve()) + def test_finds_library_in_build_dir(self, tmp_path): # Simulate the project layout: # tmp_path/bindings/python/baseRT/__init__.py diff --git a/bindings/rust/README.md b/bindings/rust/README.md index 083dda0..408ab89 100644 --- a/bindings/rust/README.md +++ b/bindings/rust/README.md @@ -20,7 +20,11 @@ By default the crates **dynamically link the one `libbaseRT.dylib`**, which already carries the Metal kernels (embedded metallib) and its own framework / C++ dependencies — one redistributable artifact, matching the Python / Node / Swift bindings. Point the build at the directory containing it with -`BASERT_LIB_DIR` (defaults to `../../../build` relative to the sys crate): +`BASERT_LIB_DIR`. When it is unset, the build uses the first of these that +holds the library: `../../../build` relative to the sys crate, a prebuilt +engine from `install.sh` (`$BASERT_INSTALL_DIR`, default `~/.basert`), or +Homebrew (`$(brew --prefix)/opt/basert/libexec`). So after +`brew install basecompute/tap/basert`, `cargo add baseRT` builds with no extra setup. ```bash export BASERT_LIB_DIR=/path/to/baseRT/build @@ -39,9 +43,24 @@ frameworks manually), enable the `static` feature: baseRT = { path = "...", features = ["static"] } ``` +A crate that loads `libbaseRT` itself at run time can depend on `baseRT-sys` +with the `no-link` feature: the build then looks for no library and links +nothing. The types stay usable; the `extern` functions must not be called. + +Either way, `baseRT-sys` tells dependents' build scripts where the engine's C +headers are: `DEP_BASERT_INCLUDE` holds `baseRT/baseRT.h`. + ## Usage -Add the dependency (path-based for now): +Add the dependency from crates.io (install the engine first, e.g. +`brew install basecompute/tap/basert`): + +```toml +[dependencies] +baseRT = "0.4" +``` + +or from a checkout: ```toml [dependencies] diff --git a/bindings/rust/baseRT-sys/Cargo.toml b/bindings/rust/baseRT-sys/Cargo.toml index d27e4ef..b132239 100644 --- a/bindings/rust/baseRT-sys/Cargo.toml +++ b/bindings/rust/baseRT-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT-sys" -version = "0.4.1" +version = "0.4.2" edition = "2021" description = "Raw FFI bindings for the BaseRT LLM inference engine" license = "Apache-2.0" @@ -12,3 +12,7 @@ default = [] # Link the static engine archive (libbaseRT_engine.a) + frameworks instead of # the single libbaseRT.dylib. Default is dynamic (one self-contained dylib). static = [] +# Link nothing: the dependent loads libbaseRT itself at run time, so it builds +# and starts without the library. The types stay usable; the extern functions +# must then not be called. +no-link = [] diff --git a/bindings/rust/baseRT-sys/build.rs b/bindings/rust/baseRT-sys/build.rs index 45bd009..dfd6218 100644 --- a/bindings/rust/baseRT-sys/build.rs +++ b/bindings/rust/baseRT-sys/build.rs @@ -1,5 +1,5 @@ use std::env; -use std::path::PathBuf; +use std::path::{Component, PathBuf}; // By default this links the single prebuilt shared library, libbaseRT.dylib, // which already carries the Metal kernels (embedded metallib) and its own @@ -8,22 +8,38 @@ use std::path::PathBuf; // // Opt into linking the static engine archive (libbaseRT_engine.a) plus the // frameworks manually with `--features static` if you need a fully static link. +// +// `--features no-link` links nothing, for a dependent that loads the library +// itself at run time. fn main() { - // Search path: BASERT_LIB_DIR (or legacy BASERT_LIB_PATH), else ../../../build. - let lib_dir = env::var("BASERT_LIB_DIR") - .or_else(|_| env::var("BASERT_LIB_PATH")) - .map(PathBuf::from) - .unwrap_or_else(|_| { - let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - manifest_dir.join("..").join("..").join("..").join("build") - }); + println!("cargo:rerun-if-env-changed=BASERT_LIB_DIR"); + println!("cargo:rerun-if-env-changed=BASERT_LIB_PATH"); + println!("cargo:rerun-if-env-changed=BASERT_INSTALL_DIR"); + + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let checkout = manifest_dir.join("..").join("..").join(".."); - let lib_dir = lib_dir.canonicalize().unwrap_or_else(|_| { - panic!( - "BaseRT library directory not found: {:?}. Set BASERT_LIB_DIR or build with `make shared`.", - lib_dir - ) - }); + // The engine's C headers, for dependents' build scripts: through `links` + // this arrives as DEP_BASERT_INCLUDE, holding baseRT/baseRT.h. The crate + // as published carries them in its own include/; in a source checkout + // they are the checkout's. Exported before anything can return early, so + // dependents get it with no-link and on docs.rs too. + let include = [manifest_dir.join("include"), checkout.join("include")] + .into_iter() + .find(|d| d.join("baseRT").join("baseRT.h").is_file()) + .and_then(|d| d.canonicalize().ok()) + .unwrap_or_else(|| manifest_dir.join("include")); + println!("cargo:include={}", include.display()); + + // docs.rs builds without the engine. The crate is only FFI declarations, + // so documenting it needs nothing to link against. + if env::var_os("DOCS_RS").is_some() { + return; + } + // The dependent loads the library at run time: nothing to find or link. + if cfg!(feature = "no-link") { + return; + } // Branch on the TARGET os, not the host: in a build script cfg!(target_os) // reflects the build script's own (host) target, so read CARGO_CFG_TARGET_OS. @@ -33,8 +49,85 @@ fn main() { "windows" => "baseRT.dll", _ => "libbaseRT.so", // linux / other unix }; + let wanted = if cfg!(feature = "static") { + "libbaseRT_engine.a" + } else { + dylib_file + }; + + // Search path: BASERT_LIB_DIR (or legacy BASERT_LIB_PATH) if set, else the + // first of these that holds the library: a source checkout's + // ../../../build, then a prebuilt engine installed by install.sh + // ($BASERT_INSTALL_DIR, default ~/.basert) or Homebrew. The checkout comes + // first so a fresh local build is never shadowed by an older release. + let lib_dir = match env::var("BASERT_LIB_DIR").or_else(|_| env::var("BASERT_LIB_PATH")) { + Ok(dir) => PathBuf::from(dir), + Err(_) => { + let mut candidates = vec![checkout.join("build")]; + match env::var_os("BASERT_INSTALL_DIR") { + Some(dir) => candidates.push(PathBuf::from(dir)), + None => { + if let Some(home) = env::var_os("HOME") { + candidates.push(PathBuf::from(home).join(".basert")); + } + } + } + candidates.push(PathBuf::from("/opt/homebrew/opt/basert/libexec")); + candidates.push(PathBuf::from( + "/home/linuxbrew/.linuxbrew/opt/basert/libexec", + )); + let found = candidates + .iter() + .find(|d| d.join(wanted).is_file()) + .cloned() + .unwrap_or_else(|| { + panic!( + "BaseRT engine library ({}) not found. Install the engine \ + (`brew install basecompute/tap/basert` or install.sh), set BASERT_LIB_DIR, \ + or build with `make shared`. Looked in: {:?}", + wanted, candidates + ) + }); + // Built inside a source checkout whose build/ has no library, this + // links an installed engine, which may be an older release than the + // checkout's headers. Say so rather than do it silently. + if found != candidates[0] && checkout.join("include/baseRT/baseRT.h").is_file() { + let build = checkout + .canonicalize() + .map_or(candidates[0].clone(), |c| c.join("build")); + println!( + "cargo:warning=no {} in {}, so the installed engine in {} is linked, \ + which may be an older release than this checkout. Build this checkout's \ + (`make shared`) or set BASERT_LIB_DIR to choose.", + wanted, + build.display(), + found.display() + ); + } + found + } + }; + + // An absolute path is kept as given, not resolved through symlinks: Homebrew's + // opt/basert/libexec is a stable link to the versioned Cellar folder, and an + // rpath into that folder would break on the next `brew upgrade`. Only a + // relative path, or one with `..` (the checkout's ../../../build), is resolved. + let keep = lib_dir.is_absolute() && !lib_dir.components().any(|c| c == Component::ParentDir); + let lib_dir = if keep { Ok(lib_dir.clone()) } else { lib_dir.canonicalize() } + .ok() + .filter(|d| d.is_dir()) + .unwrap_or_else(|| { + panic!( + "BaseRT library directory not found: {:?}. Set BASERT_LIB_DIR or build with `make shared`.", + lib_dir + ) + }); println!("cargo:rustc-link-search=native={}", lib_dir.display()); + // Hand the resolved directory to crates that depend on this one (via + // `links = "baseRT"` it arrives as DEP_BASERT_LIB_DIR), so baseRT embeds + // an rpath to the same place without repeating this search. + println!("cargo:lib_dir={}", lib_dir.display()); if cfg!(feature = "static") { // Static engine + the platform deps it pulls in. @@ -67,7 +160,4 @@ fn main() { lib_dir.join(dylib_file).display() ); } - - println!("cargo:rerun-if-env-changed=BASERT_LIB_DIR"); - println!("cargo:rerun-if-env-changed=BASERT_LIB_PATH"); } diff --git a/bindings/rust/baseRT/Cargo.toml b/bindings/rust/baseRT/Cargo.toml index 24b16fb..3cd2300 100644 --- a/bindings/rust/baseRT/Cargo.toml +++ b/bindings/rust/baseRT/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT" -version = "0.4.1" +version = "0.4.2" edition = "2021" description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)" license = "Apache-2.0" diff --git a/bindings/rust/baseRT/build.rs b/bindings/rust/baseRT/build.rs index f4ac7a8..45c8612 100644 --- a/bindings/rust/baseRT/build.rs +++ b/bindings/rust/baseRT/build.rs @@ -1,5 +1,4 @@ use std::env; -use std::path::PathBuf; // baseRT-sys emits the link-search + link-lib (those propagate to us), but // Cargo does NOT propagate a dependency's `rustc-link-arg`, so the rpath it @@ -7,22 +6,19 @@ use std::path::PathBuf; // here so `cargo run --example` / `cargo test` find libbaseRT.dylib at runtime // without DYLD_LIBRARY_PATH. (Downstream user crates that build executables // should do the same, or set an rpath / DYLD_LIBRARY_PATH — see README.) +// +// The directory comes from baseRT-sys, which resolves it (BASERT_LIB_DIR, a +// source checkout's build/, ~/.basert or Homebrew) and passes it on through its +// `links` metadata as DEP_BASERT_LIB_DIR. It is unset on docs.rs, where +// baseRT-sys links nothing. fn main() { - let lib_dir = env::var("BASERT_LIB_DIR") - .or_else(|_| env::var("BASERT_LIB_PATH")) - .map(PathBuf::from) - .unwrap_or_else(|_| { - let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - manifest_dir.join("..").join("..").join("..").join("build") - }); + println!("cargo:rerun-if-env-changed=DEP_BASERT_LIB_DIR"); // Windows resolves DLLs via PATH / the exe directory, not rpath. let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); if target_os != "windows" { - if let Ok(dir) = lib_dir.canonicalize() { - println!("cargo:rustc-link-arg=-Wl,-rpath,{}", dir.display()); + if let Ok(dir) = env::var("DEP_BASERT_LIB_DIR") { + println!("cargo:rustc-link-arg=-Wl,-rpath,{dir}"); } } - println!("cargo:rerun-if-env-changed=BASERT_LIB_DIR"); - println!("cargo:rerun-if-env-changed=BASERT_LIB_PATH"); } diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index 2774b34..cc5bec9 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 1 +#define BASERT_VERSION_PATCH 2 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index 2774b34..cc5bec9 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 1 +#define BASERT_VERSION_PATCH 2 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. diff --git a/packaging/npm/build-packages.mjs b/packaging/npm/build-packages.mjs index 0971eee..c7e7adc 100644 --- a/packaging/npm/build-packages.mjs +++ b/packaging/npm/build-packages.mjs @@ -19,8 +19,7 @@ // (default: all, `basert-darwin-arm64,basert-linux-arm64-cuda`). Every engine // tarball is still built, for testing, but the main package depends only on // the named ones: one that never reaches the registry must not be among its -// optionalDependencies. publish.yml leaves the Linux engine out until its -// licence covers Linux. +// optionalDependencies. publish.yml publishes all of them. // // bindings/node is compiled in a staging copy and its package.json rewritten // there, so the published package gets its own name and metadata while the @@ -127,11 +126,19 @@ try { } // --- main package: compile the bindings in a staging copy ----------------- + // The copy's package.json leaves out os, cpu and libc. Those say where the + // bindings install (darwin/arm64), and npm ci refuses a project whose own + // platforms exclude the machine it runs on, such as publish.yml's Linux x64 + // build runner. Compiling needs none of them; the main package below states + // its own. const build = path.join(stage, "build"); fs.mkdirSync(build); - for (const f of ["src", "tsconfig.json", "package.json", "package-lock.json"]) { + for (const f of ["src", "tsconfig.json", "package-lock.json"]) { fs.cpSync(path.join(root, "bindings/node", f), path.join(build, f), { recursive: true }); } + const buildable = { ...source }; + for (const key of ["os", "cpu", "libc"]) delete buildable[key]; + writeJson(path.join(build, "package.json"), buildable); run("npm", ["ci", "--ignore-scripts", "--no-audit", "--no-fund"], build); run("npx", ["tsc"], build); diff --git a/tests/scripts/test_packaging.py b/tests/scripts/test_packaging.py index 0fd3813..4a840b2 100644 --- a/tests/scripts/test_packaging.py +++ b/tests/scripts/test_packaging.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Unit tests for packaging/stage-engine.sh and packaging/check_engine_links.py. +"""Unit tests for packaging/stage-engine.sh, packaging/check_engine_links.py and +the npm launcher and package build (packaging/npm/). The bundles here are synthetic: a library is a few bytes of text, and the Mach-O and ELF programs carry only the load commands the link check reads. @@ -276,5 +277,102 @@ def test_an_engine_this_release_did_not_publish_points_at_install_sh(self) -> No self.assertIn("this release has no npm engine package", run.stderr) +class NpmBuildTests(unittest.TestCase): + """packaging/npm/build-packages.mjs, run with npm over synthetic bundles. + + The bindings it compiles say which platforms they install on, and + publish.yml builds on one they leave out (Linux x64). Here they name a + platform no test machine is, which `npm ci` refuses unless the build + compiles a copy without it. The bindings have no dependencies and `npx tsc` + is stubbed, so nothing is fetched. + """ + + VERSION = "1.2.3" + FOREIGN = {"os": ["win32"], "cpu": ["ia32"]} + + def setUp(self) -> None: + node = shutil.which("node") + if not node or not shutil.which("npm"): + self.skipTest("needs node and npm") + self.node = node + self.tmp = Path(tempfile.mkdtemp()) + v = self.VERSION + # A public tree with just what the build reads. + self.root = self.tmp / "tree" + (self.root / "packaging" / "npm").mkdir(parents=True) + for name in ("npm/build-packages.mjs", "npm/basert.js", "npm/README.md", "stage-engine.sh", "check_engine_links.py"): + shutil.copy(ROOT / "packaging" / name, self.root / "packaging" / name) + (self.root / "LICENSE").write_text("licence") + (self.root / "NOTICE").write_text("notice") + bindings = self.root / "bindings" / "node" + (bindings / "src").mkdir(parents=True) + (bindings / "src" / "index.ts").write_text("export {};\n") + (bindings / "tsconfig.json").write_text("{}\n") + (bindings / "package.json").write_text( + json.dumps({"name": "@baseRT/node", "version": v, "main": "dist/index.js", **self.FOREIGN}) + ) + (bindings / "package-lock.json").write_text( + json.dumps( + { + "name": "@baseRT/node", + "version": v, + "lockfileVersion": 3, + "requires": True, + "packages": {"": {"name": "@baseRT/node", "version": v, **self.FOREIGN}}, + } + ) + ) + # Stands in for `npx tsc` compiling src/index.ts. + self.bin = self.tmp / "bin" + self.bin.mkdir() + (self.bin / "npx").write_text( + '#!/bin/sh\n[ "$1" = tsc ] || exit 1\nmkdir -p dist\necho "exports.v = 1;" > dist/index.js\n' + ) + os.chmod(self.bin / "npx", 0o755) + self.bundles = self.tmp / "bundles" + self.bundles.mkdir() + bundle( + self.bundles / f"basert-engine-macos-arm64-{v}.tar.gz", + {f"libbaseRT.{v}.dylib": macho([], own_id="@rpath/libbaseRT.0.dylib"), "basert": macho([]), "LICENSE": b"engine"}, + {"libbaseRT.0.dylib": f"libbaseRT.{v}.dylib", "libbaseRT.dylib": "libbaseRT.0.dylib"}, + ) + bundle( + self.bundles / f"basert-engine-linux-arm64-cuda-{v}.tar.gz", + {f"libbaseRT.so.{v}": elf([], soname="libbaseRT.so.0"), "basert": elf([]), "LICENSE": b"engine"}, + {"libbaseRT.so.0": f"libbaseRT.so.{v}", "libbaseRT.so": "libbaseRT.so.0"}, + ) + + def build(self, *args: str) -> tuple[subprocess.CompletedProcess, Path]: + out = self.tmp / "out" + env = dict( + os.environ, + PATH=f"{self.bin}{os.pathsep}{os.environ['PATH']}", + npm_config_cache=str(self.tmp / "npm-cache"), + npm_config_update_notifier="false", + ) + script = self.root / "packaging" / "npm" / "build-packages.mjs" + run = subprocess.run( + [self.node, str(script), str(self.bundles), str(out), *args], capture_output=True, text=True, env=env + ) + return run, out + + def test_the_bindings_compile_on_a_machine_they_do_not_install_on(self) -> None: + run, out = self.build("--engines", "basert-darwin-arm64") + self.assertEqual(run.returncode, 0, run.stderr) + v = self.VERSION + self.assertEqual( + sorted(p.name for p in out.iterdir()), + [f"basecompute-basert-{v}.tgz", f"basecompute-basert-darwin-arm64-{v}.tgz", f"basecompute-basert-linux-arm64-cuda-{v}.tgz"], + ) + with tarfile.open(out / f"basecompute-basert-{v}.tgz") as tar: + main = json.load(tar.extractfile("package/package.json")) + self.assertEqual((main["os"], main["cpu"]), (["darwin", "linux"], ["arm64"])) + # Only the engine package this release publishes. + self.assertEqual(main["optionalDependencies"], {"@basecompute/basert-darwin-arm64": v}) + # The bindings' own folder is left as it was. + source = json.loads((self.root / "bindings" / "node" / "package.json").read_text()) + self.assertEqual(source["os"], self.FOREIGN["os"]) + + if __name__ == "__main__": unittest.main()