diff --git a/Cargo.lock b/Cargo.lock index 7bb2a2c17..9186550d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -190,7 +190,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -1171,15 +1171,14 @@ dependencies = [ "llama-cpp-bindings-sys", "llama-cpp-bindings-types", "llama-cpp-error-recorder", + "llama-cpp-fault-injection", "llama-cpp-ffi-status", "llama-cpp-gbnf", "llama-cpp-log-decoder", "llama-cpp-wrapper-error-fixture", "llguidance", "log", - "nom 8.0.0", "once_cell", - "serde", "serde_json", "serial_test", "thiserror", @@ -1193,8 +1192,11 @@ dependencies = [ "bindgen", "cc", "cmake", + "enumflags2", "find_cuda_helper", "llama-cpp-wrapper-sources", + "serde", + "serde_json", "thiserror", ] @@ -1211,10 +1213,16 @@ version = "0.26.0" dependencies = [ "anyhow", "encoding_rs", + "inventory", "llama-cpp-bindings", "llama-cpp-bindings-sys", "llama-cpp-bindings-types", + "llama-cpp-fault-injection", + "llama-cpp-ffi-status", + "llama-cpp-gbnf", "llama-cpp-test-harness", + "llama-cpp-test-harness-macros", + "llama-cpp-test-model-provisioning", "serde_json", "toktrie", ] @@ -1235,6 +1243,13 @@ dependencies = [ "log", ] +[[package]] +name = "llama-cpp-fault-injection" +version = "0.26.0" +dependencies = [ + "llama-cpp-bindings-sys", +] + [[package]] name = "llama-cpp-ffi-status" version = "0.26.0" @@ -1262,11 +1277,11 @@ name = "llama-cpp-test-harness" version = "0.26.0" dependencies = [ "anyhow", - "hf-hub", "inventory", "libtest-mimic", "llama-cpp-bindings", "llama-cpp-test-harness-macros", + "llama-cpp-test-model-provisioning", "thiserror", ] @@ -1279,6 +1294,15 @@ dependencies = [ "syn", ] +[[package]] +name = "llama-cpp-test-model-provisioning" +version = "0.26.0" +dependencies = [ + "hf-hub", + "llama-cpp-bindings", + "thiserror", +] + [[package]] name = "llama-cpp-wrapper-error-fixture" version = "0.26.0" @@ -1393,15 +1417,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - [[package]] name = "num-conv" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 1c2f6e953..01301246b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,11 +7,13 @@ members = [ "llama-cpp-bindings", "llama-cpp-bindings-tests", "llama-cpp-error-recorder", + "llama-cpp-fault-injection", "llama-cpp-ffi-status", "llama-cpp-gbnf", "llama-cpp-log-decoder", "llama-cpp-test-harness", "llama-cpp-test-harness-macros", + "llama-cpp-test-model-provisioning", "llama-cpp-wrapper-error-fixture", "llama-cpp-wrapper-sources", ] @@ -43,11 +45,12 @@ llama-cpp-gbnf = { path = "llama-cpp-gbnf", version = "=0.26.0" } llama-cpp-log-decoder = { path = "llama-cpp-log-decoder", version = "=0.26.0" } llama-cpp-test-harness = { path = "llama-cpp-test-harness", version = "=0.26.0" } llama-cpp-test-harness-macros = { path = "llama-cpp-test-harness-macros", version = "=0.26.0" } +llama-cpp-test-model-provisioning = { path = "llama-cpp-test-model-provisioning", version = "=0.26.0" } +llama-cpp-fault-injection = { path = "llama-cpp-fault-injection" } llama-cpp-wrapper-error-fixture = { path = "llama-cpp-wrapper-error-fixture" } llama-cpp-wrapper-sources = { path = "llama-cpp-wrapper-sources", version = "=0.26.0" } llguidance = "=1.7.0" log = "=0.4.29" -nom = "=8.0.0" once_cell = "1.21" proc-macro2 = "=1.0.106" quote = "=1.0.45" diff --git a/Makefile b/Makefile index 4b21a902d..a20ec822f 100644 --- a/Makefile +++ b/Makefile @@ -19,12 +19,8 @@ WRAPPER_SOURCES_RESPONSE_FILE = target/wrapper_sources.rsp WRAPPER_SOURCES_CRATE = llama-cpp-wrapper-sources WRAPPER_SOURCES_CRATE_FILES = \ - $(WRAPPER_SOURCES_CRATE)/src/compile_commands_file.rs \ - $(WRAPPER_SOURCES_CRATE)/src/cpp_standard.rs \ - $(WRAPPER_SOURCES_CRATE)/src/wrapper_include_dirs.rs \ - $(WRAPPER_SOURCES_CRATE)/src/wrapper_source_paths.rs \ - $(WRAPPER_SOURCES_CRATE)/src/wrapper_sources.rs \ - $(WRAPPER_SOURCES_CRATE)/src/wrapper_sources_response_file.rs + $(WRAPPER_SOURCES_CRATE)/Cargo.toml \ + $(wildcard $(WRAPPER_SOURCES_CRATE)/src/*.rs) EMIT_WRAPPER_BUILD_INPUTS = cargo run --quiet --package $(WRAPPER_SOURCES_CRATE) -- \ $(CURDIR)/llama-cpp-bindings-sys $(COMPILE_COMMANDS) $(WRAPPER_SOURCES_RESPONSE_FILE) @@ -76,14 +72,18 @@ coverage: node_modules cargo llvm-cov report ./node_modules/.bin/rust-coverage-check target/llvm-cov.json \ --workspace-root $(CURDIR) \ - --gated llama-cpp-bindings=98 \ + --gated llama-cpp-bindings=100 \ + --gated llama-cpp-bindings-build=100 \ + --gated llama-cpp-bindings-tests=100 \ --gated llama-cpp-bindings-types=100 \ --gated llama-cpp-error-recorder=100 \ + --gated llama-cpp-fault-injection=100 \ --gated llama-cpp-ffi-status=100 \ --gated llama-cpp-gbnf=100 \ --gated llama-cpp-log-decoder=100 \ - --gated llama-cpp-test-harness=99 \ + --gated llama-cpp-test-harness=100 \ --gated llama-cpp-test-harness-macros=100 \ + --gated llama-cpp-test-model-provisioning=100 \ --gated llama-cpp-wrapper-sources=100 .PHONY: coverage-clean @@ -123,7 +123,8 @@ test: test.llms .PHONY: test.harness test.harness: clippy - cargo test -p llama-cpp-test-harness-macros -p llama-cpp-test-harness $(DEVICE_FEATURE) + cargo test -p llama-cpp-test-harness-macros -p llama-cpp-test-harness \ + -p llama-cpp-test-model-provisioning $(DEVICE_FEATURE) .PHONY: test.llms test.llms: clippy test.harness test.unit @@ -131,5 +132,6 @@ test.llms: clippy test.harness test.unit .PHONY: test.unit test.unit: clippy - cargo test -p llama-cpp-bindings -p llama-cpp-bindings-build -p llama-cpp-gbnf \ + cargo test -p llama-cpp-bindings -p llama-cpp-bindings-build -p llama-cpp-bindings-types \ + -p llama-cpp-error-recorder -p llama-cpp-ffi-status -p llama-cpp-gbnf \ -p llama-cpp-log-decoder -p llama-cpp-wrapper-sources $(DEVICE_FEATURE) diff --git a/llama-cpp-bindings-build/Cargo.toml b/llama-cpp-bindings-build/Cargo.toml index 34039785d..3aae18bbd 100644 --- a/llama-cpp-bindings-build/Cargo.toml +++ b/llama-cpp-bindings-build/Cargo.toml @@ -10,8 +10,11 @@ repository.workspace = true bindgen = { workspace = true } cc = { workspace = true } cmake = { workspace = true } +enumflags2 = { workspace = true } find_cuda_helper = { workspace = true } llama-cpp-wrapper-sources = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } [features] @@ -20,6 +23,7 @@ cuda-no-vmm = ["cuda"] metal = [] dynamic-link = [] dynamic-backends = ["dynamic-link"] +fault-injection = [] vulkan = [] openmp = [] rocm = [] diff --git a/llama-cpp-bindings-build/cmake/cache_query.cmake b/llama-cpp-bindings-build/cmake/cache_query.cmake new file mode 100644 index 000000000..590718d7d --- /dev/null +++ b/llama-cpp-bindings-build/cmake/cache_query.cmake @@ -0,0 +1 @@ +file(WRITE "${CMAKE_BINARY_DIR}/.cmake/api/v1/query/cache-v2" "") diff --git a/llama-cpp-bindings-build/fixtures/android_ndk/build/cmake/android.toolchain.cmake b/llama-cpp-bindings-build/fixtures/android_ndk/build/cmake/android.toolchain.cmake new file mode 100644 index 000000000..e69de29bb diff --git a/llama-cpp-bindings-build/fixtures/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/20/include/stddef.h b/llama-cpp-bindings-build/fixtures/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/20/include/stddef.h new file mode 100644 index 000000000..e69de29bb diff --git a/llama-cpp-bindings-build/fixtures/bindings/llama_types.h b/llama-cpp-bindings-build/fixtures/bindings/llama_types.h new file mode 100644 index 000000000..9f275522b --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/bindings/llama_types.h @@ -0,0 +1,20 @@ +#pragma once + +struct llama_context_params { + float defrag_thold; + unsigned int n_ctx; +}; + +struct llama_token_data { + int id; + float logit; +}; + +struct llama_sampler_chain_params { + int no_perf; +}; + +enum ggml_prec { + GGML_PREC_UNDEFINED = 0, + GGML_PREC_DEFAULT = 0, +}; diff --git a/llama-cpp-bindings-build/fixtures/bindings/missing_privatized_field.h b/llama-cpp-bindings-build/fixtures/bindings/missing_privatized_field.h new file mode 100644 index 000000000..5429a180b --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/bindings/missing_privatized_field.h @@ -0,0 +1,5 @@ +#pragma once + +struct llama_context_params { + unsigned int n_ctx; +}; diff --git a/llama-cpp-bindings-build/fixtures/bindings/wrapper.h b/llama-cpp-bindings-build/fixtures/bindings/wrapper.h new file mode 100644 index 000000000..8049a548d --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/bindings/wrapper.h @@ -0,0 +1,8 @@ +#pragma once + +#include "llama_types.h" + +struct mtmd_context_params { + const char * image_marker; + const char * media_marker; +}; diff --git a/llama-cpp-bindings-build/fixtures/cmake_file_api/index_naming_a_missing_cache/.cmake/api/v1/reply/index-0.json b/llama-cpp-bindings-build/fixtures/cmake_file_api/index_naming_a_missing_cache/.cmake/api/v1/reply/index-0.json new file mode 100644 index 000000000..1d183cfb6 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_file_api/index_naming_a_missing_cache/.cmake/api/v1/reply/index-0.json @@ -0,0 +1 @@ +{"reply":{"cache-v2":{"kind":"cache","version":{"major":2,"minor":0},"jsonFile":"cache-v2-missing.json"}}} diff --git a/llama-cpp-bindings-build/fixtures/cmake_file_api/index_without_cache_reply/.cmake/api/v1/reply/index-0.json b/llama-cpp-bindings-build/fixtures/cmake_file_api/index_without_cache_reply/.cmake/api/v1/reply/index-0.json new file mode 100644 index 000000000..ee5d64f1f --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_file_api/index_without_cache_reply/.cmake/api/v1/reply/index-0.json @@ -0,0 +1 @@ +{"reply":{}} diff --git a/llama-cpp-bindings-build/fixtures/cmake_file_api/reply_without_index/.cmake/api/v1/reply/cache-v2-0.json b/llama-cpp-bindings-build/fixtures/cmake_file_api/reply_without_index/.cmake/api/v1/reply/cache-v2-0.json new file mode 100644 index 000000000..f5cae2c6b --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_file_api/reply_without_index/.cmake/api/v1/reply/cache-v2-0.json @@ -0,0 +1 @@ +{"kind":"cache","version":{"major":2,"minor":0},"entries":[]} diff --git a/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt b/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt new file mode 100644 index 000000000..b58105960 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_project/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.14) +project(llama_cpp_bindings_build_fixture C) + +if(NOT DEFINED FIXTURE_LIBRARY_NAME) + message(FATAL_ERROR "FIXTURE_LIBRARY_NAME must be defined") +endif() + +add_library(${FIXTURE_LIBRARY_NAME} STATIC fixture.c) + +install(TARGETS ${FIXTURE_LIBRARY_NAME} DESTINATION lib) + +set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "") +set(OpenMP_C_LIB_NAMES "${FIXTURE_LIBRARY_NAME}" CACHE STRING "") +set(OpenMP_CXX_LIB_NAMES "${FIXTURE_LIBRARY_NAME}" CACHE STRING "") +set(OpenMP_${FIXTURE_LIBRARY_NAME}_LIBRARY + "${CMAKE_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${FIXTURE_LIBRARY_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + CACHE FILEPATH "") diff --git a/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c b/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c new file mode 100644 index 000000000..f53a279e2 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/cmake_project/fixture.c @@ -0,0 +1,3 @@ +int llama_cpp_bindings_build_fixture(void) { + return 0; +} diff --git a/llama-cpp-bindings-build/fixtures/wrapper/vendor/vendored.h b/llama-cpp-bindings-build/fixtures/wrapper/vendor/vendored.h new file mode 100644 index 000000000..9e324d736 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/wrapper/vendor/vendored.h @@ -0,0 +1,3 @@ +#pragma once + +#define VENDORED_VALUE 7 diff --git a/llama-cpp-bindings-build/fixtures/wrapper/wrapper_fixture.cpp b/llama-cpp-bindings-build/fixtures/wrapper/wrapper_fixture.cpp new file mode 100644 index 000000000..ec19a8058 --- /dev/null +++ b/llama-cpp-bindings-build/fixtures/wrapper/wrapper_fixture.cpp @@ -0,0 +1,9 @@ +#include + +#ifndef WRAPPER_FIXTURE_DEFINE +#error "WRAPPER_FIXTURE_DEFINE must be defined" +#endif + +extern "C" auto llama_cpp_bindings_build_wrapper_fixture() -> int { + return VENDORED_VALUE; +} diff --git a/llama-cpp-bindings-build/src/android_architecture.rs b/llama-cpp-bindings-build/src/android_architecture.rs new file mode 100644 index 000000000..69c744e5b --- /dev/null +++ b/llama-cpp-bindings-build/src/android_architecture.rs @@ -0,0 +1,69 @@ +use crate::build_error::BuildError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AndroidArchitecture { + Aarch64, + Armv7, + X86_64, + X86, +} + +impl AndroidArchitecture { + /// # Errors + /// + /// Returns [`BuildError::UnsupportedAndroidTarget`] for an architecture the NDK does not + /// build for. + pub fn from_cargo_cfg(cargo_cfg_target_arch: &str) -> Result { + match cargo_cfg_target_arch { + "aarch64" => Ok(Self::Aarch64), + "arm" => Ok(Self::Armv7), + "x86_64" => Ok(Self::X86_64), + "x86" => Ok(Self::X86), + unsupported => Err(BuildError::UnsupportedAndroidTarget { + cargo_cfg_target_arch: unsupported.to_owned(), + }), + } + } + + #[must_use] + pub const fn abi(self) -> &'static str { + match self { + Self::Aarch64 => "arm64-v8a", + Self::Armv7 => "armeabi-v7a", + Self::X86_64 => "x86_64", + Self::X86 => "x86", + } + } + + #[must_use] + pub const fn ndk_prefix(self) -> &'static str { + match self { + Self::Aarch64 => "aarch64-linux-android", + Self::Armv7 => "arm-linux-androideabi", + Self::X86_64 => "x86_64-linux-android", + Self::X86 => "i686-linux-android", + } + } +} + +#[cfg(test)] +mod tests { + use super::AndroidArchitecture; + + #[test] + fn every_supported_android_architecture_maps_to_its_abi_and_ndk_prefix() { + assert_eq!( + ["aarch64", "arm", "x86_64", "x86"].map(|cargo_cfg_target_arch| { + AndroidArchitecture::from_cargo_cfg(cargo_cfg_target_arch) + .map(|architecture| [architecture.abi(), architecture.ndk_prefix()]) + .ok() + }), + [ + Some(["arm64-v8a", "aarch64-linux-android"]), + Some(["armeabi-v7a", "arm-linux-androideabi"]), + Some(["x86_64", "x86_64-linux-android"]), + Some(["x86", "i686-linux-android"]), + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/android_ndk.rs b/llama-cpp-bindings-build/src/android_ndk.rs index d6734fbc4..39d2fffd1 100644 --- a/llama-cpp-bindings-build/src/android_ndk.rs +++ b/llama-cpp-bindings-build/src/android_ndk.rs @@ -1,58 +1,103 @@ -use std::env; -use std::path::{Path, PathBuf}; - -use thiserror::Error; +use std::env::VarError; +use std::fs; +use std::fs::DirEntry; +use std::io; +use std::path::Path; +use std::path::PathBuf; + +use crate::android_architecture::AndroidArchitecture; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::host_os::HostOs; const DEFAULT_ANDROID_API_LEVEL: &str = "28"; -#[derive(Debug, Error)] -pub enum AndroidNdkDetectionError { - #[error("Android NDK not found for target {target_triple}. Set ANDROID_NDK_HOME.")] - NdkRootNotConfigured { - target_triple: String, - #[source] - source: env::VarError, - }, - #[error("Android NDK path does not exist: {path}")] - NdkRootMissing { path: PathBuf }, - #[error("Android NDK toolchain file not found: {path}")] - NdkToolchainFileMissing { path: PathBuf }, - #[error("Android NDK toolchain not found at: {path}")] - NdkToolchainDirectoryMissing { path: PathBuf }, - #[error("Unsupported host platform for Android NDK")] - UnsupportedHostPlatform, - #[error("Unsupported Android target architecture: {cargo_cfg_target_arch}")] - UnsupportedAndroidTarget { cargo_cfg_target_arch: String }, - #[error("ANDROID_PLATFORM is set but could not be read: {source}")] - AndroidPlatformUnreadable { - #[source] - source: env::VarError, - }, - #[error("Android NDK Clang directory could not be read at {path}: {source}")] - ClangDirectoryUnreadable { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("Android NDK Clang version directory was not found at {path}")] - ClangVersionDirectoryMissing { path: PathBuf }, - #[error("Android NDK Clang directory entry could not be read at {path}: {source}")] - ClangDirectoryEntryUnreadable { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("Android NDK Clang built-in include directory was not found at {path}")] - ClangBuiltinIncludesMissing { path: PathBuf }, +fn detect_ndk_path( + environment: &BuildEnvironment, + target_triple: &str, +) -> Result { + environment + .var("ANDROID_NDK_HOME") + .map_err(|source| BuildError::AndroidNdkRootNotConfigured { + target_triple: target_triple.to_owned(), + source, + }) +} + +fn validate_ndk_installation(ndk_path: &str) -> Result<(), BuildError> { + let ndk_path = Path::new(ndk_path); + + if !ndk_path.exists() { + return Err(BuildError::AndroidNdkRootMissing { + path: ndk_path.to_path_buf(), + }); + } + + let toolchain_file = ndk_path.join("build/cmake/android.toolchain.cmake"); + + if !toolchain_file.exists() { + return Err(BuildError::AndroidNdkToolchainFileMissing { + path: toolchain_file, + }); + } + + Ok(()) +} + +fn detect_api_level(environment: &BuildEnvironment) -> Result { + match environment.var("ANDROID_PLATFORM") { + Ok(platform) => Ok(platform + .strip_prefix("android-") + .unwrap_or(&platform) + .to_owned()), + Err(VarError::NotPresent) => Ok(DEFAULT_ANDROID_API_LEVEL.to_owned()), + Err(source) => Err(BuildError::AndroidPlatformUnreadable { source }), + } +} + +/// The NDK toolchain keeps the built-in headers of its clang in the single version directory +/// under `lib/clang`. +fn find_clang_builtin_includes(toolchain_path: &str) -> Result { + let clang_lib_path = PathBuf::from(toolchain_path).join("lib/clang"); + let mut version_dirs = fs::read_dir(&clang_lib_path) + .and_then(Iterator::collect::, io::Error>>) + .map_err(|source| BuildError::AndroidNdkClangDirectoryUnreadable { + path: clang_lib_path.clone(), + source, + })? + .into_iter() + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .collect::>(); + + version_dirs.sort(); + + let [version_dir] = <[PathBuf; 1]>::try_from(version_dirs).map_err(|candidates| { + if candidates.is_empty() { + BuildError::AndroidNdkClangVersionDirectoryMissing { + path: clang_lib_path.clone(), + } + } else { + BuildError::AndroidNdkClangVersionDirectoryAmbiguous { + path: clang_lib_path.clone(), + candidates, + } + } + })?; + let include_path = version_dir.join("include"); + + if !include_path.is_dir() { + return Err(BuildError::AndroidNdkClangBuiltinIncludesMissing { path: include_path }); + } + + Ok(include_path.to_string_lossy().into_owned()) } -#[derive(Debug)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct AndroidNdk { pub ndk_path: String, pub api_level: String, pub abi: &'static str, - pub host_tag: &'static str, - pub toolchain_path: String, pub sysroot: String, pub target_prefix: &'static str, pub clang_builtin_includes: String, @@ -61,362 +106,356 @@ pub struct AndroidNdk { impl AndroidNdk { /// # Errors /// - /// Returns [`AndroidNdkDetectionError`] when the NDK installation cannot be - /// located, an environment variable is missing or malformed, the target - /// architecture is unsupported, or the host platform is not supported by the NDK. + /// Returns the [`BuildError`] describing why the NDK installation cannot be located, an + /// environment variable is missing or malformed, the target architecture is unsupported, or + /// the host platform is not supported by the NDK. pub fn detect( + environment: &BuildEnvironment, target_triple: &str, cargo_cfg_target_arch: &str, - ) -> Result { - let ndk_path = detect_ndk_path(target_triple)?; + host_os: HostOs, + ) -> Result { + let ndk_path = detect_ndk_path(environment, target_triple)?; validate_ndk_installation(&ndk_path)?; let architecture = AndroidArchitecture::from_cargo_cfg(cargo_cfg_target_arch)?; - let api_level = detect_api_level()?; - let abi = architecture.abi(); - let host_tag = detect_host_tag()?; - let target_prefix = architecture.ndk_prefix(); + let api_level = detect_api_level(environment)?; + let host_tag = host_os.android_ndk_tag()?; let toolchain_path = format!("{ndk_path}/toolchains/llvm/prebuilt/{host_tag}"); if !Path::new(&toolchain_path).exists() { - return Err(AndroidNdkDetectionError::NdkToolchainDirectoryMissing { + return Err(BuildError::AndroidNdkToolchainDirectoryMissing { path: PathBuf::from(toolchain_path), }); } - let sysroot = format!("{toolchain_path}/sysroot"); let clang_builtin_includes = find_clang_builtin_includes(&toolchain_path)?; Ok(Self { ndk_path, api_level, - abi, - host_tag, - toolchain_path, - sysroot, - target_prefix, + abi: architecture.abi(), + sysroot: format!("{toolchain_path}/sysroot"), + target_prefix: architecture.ndk_prefix(), clang_builtin_includes, }) } + #[must_use] pub fn android_platform(&self) -> String { format!("android-{}", self.api_level) } + #[must_use] pub fn cmake_toolchain_file(&self) -> String { format!("{}/build/cmake/android.toolchain.cmake", self.ndk_path) } -} - -fn detect_ndk_path(target_triple: &str) -> Result { - env::var("ANDROID_NDK_HOME").map_err(|source| AndroidNdkDetectionError::NdkRootNotConfigured { - target_triple: target_triple.to_owned(), - source, - }) -} - -fn validate_ndk_installation(ndk_path: &str) -> Result<(), AndroidNdkDetectionError> { - let ndk_path = Path::new(ndk_path); - - if !ndk_path.exists() { - return Err(AndroidNdkDetectionError::NdkRootMissing { - path: ndk_path.to_path_buf(), - }); - } - - let toolchain_file = ndk_path.join("build/cmake/android.toolchain.cmake"); - - if !toolchain_file.exists() { - return Err(AndroidNdkDetectionError::NdkToolchainFileMissing { - path: toolchain_file, - }); - } - - Ok(()) -} - -fn detect_api_level() -> Result { - match env::var("ANDROID_PLATFORM") { - Ok(platform) => Ok(platform - .strip_prefix("android-") - .unwrap_or(&platform) - .to_owned()), - Err(env::VarError::NotPresent) => Ok(DEFAULT_ANDROID_API_LEVEL.to_string()), - Err(source) => Err(AndroidNdkDetectionError::AndroidPlatformUnreadable { source }), - } -} - -const fn detect_host_tag() -> Result<&'static str, AndroidNdkDetectionError> { - if cfg!(target_os = "macos") { - Ok("darwin-x86_64") - } else if cfg!(target_os = "linux") { - Ok("linux-x86_64") - } else { - Err(AndroidNdkDetectionError::UnsupportedHostPlatform) - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum AndroidArchitecture { - Aarch64, - Armv7, - X86_64, - X86, -} - -impl AndroidArchitecture { - fn from_cargo_cfg(cargo_cfg_target_arch: &str) -> Result { - match cargo_cfg_target_arch { - "aarch64" => Ok(Self::Aarch64), - "arm" => Ok(Self::Armv7), - "x86_64" => Ok(Self::X86_64), - "x86" => Ok(Self::X86), - unsupported => Err(AndroidNdkDetectionError::UnsupportedAndroidTarget { - cargo_cfg_target_arch: unsupported.to_owned(), - }), - } - } - - const fn abi(self) -> &'static str { - match self { - Self::Aarch64 => "arm64-v8a", - Self::Armv7 => "armeabi-v7a", - Self::X86_64 => "x86_64", - Self::X86 => "x86", - } - } - const fn ndk_prefix(self) -> &'static str { - match self { - Self::Aarch64 => "aarch64-linux-android", - Self::Armv7 => "arm-linux-androideabi", - Self::X86_64 => "x86_64-linux-android", - Self::X86 => "i686-linux-android", - } + /// The clang arguments that parse headers the way the NDK compiles them for `target_triple`. + #[must_use] + pub fn clang_args(&self, target_triple: &str) -> Vec { + vec![ + format!("--sysroot={}", self.sysroot), + format!("-D__ANDROID_API__={}", self.api_level), + "-D__ANDROID__".to_owned(), + "-isystem".to_owned(), + self.clang_builtin_includes.clone(), + "-isystem".to_owned(), + format!("{}/usr/include/{}", self.sysroot, self.target_prefix), + "-isystem".to_owned(), + format!("{}/usr/include", self.sysroot), + "-include".to_owned(), + "stdbool.h".to_owned(), + "-include".to_owned(), + "stdint.h".to_owned(), + format!("--target={target_triple}"), + ] } } -fn find_clang_builtin_includes(toolchain_path: &str) -> Result { - let clang_lib_path = PathBuf::from(toolchain_path).join("lib/clang"); - let entries = std::fs::read_dir(&clang_lib_path).map_err(|source| { - AndroidNdkDetectionError::ClangDirectoryUnreadable { - path: clang_lib_path.clone(), - source, - } - })?; - let mut version_dir = None; - for entry in entries { - let entry = - entry.map_err( - |source| AndroidNdkDetectionError::ClangDirectoryEntryUnreadable { - path: clang_lib_path.clone(), - source, - }, - )?; - let file_type = entry.file_type().map_err(|source| { - AndroidNdkDetectionError::ClangDirectoryEntryUnreadable { - path: entry.path(), - source, - } - })?; - if file_type.is_dir() - && entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(|character: char| character.is_ascii_digit())) - { - version_dir = Some(entry); - break; - } - } - let version_dir = - version_dir.ok_or_else(|| AndroidNdkDetectionError::ClangVersionDirectoryMissing { - path: clang_lib_path.clone(), - })?; - - let include_path = clang_lib_path.join(version_dir.file_name()).join("include"); - - if !include_path.is_dir() { - return Err(AndroidNdkDetectionError::ClangBuiltinIncludesMissing { path: include_path }); - } - - Ok(include_path.to_string_lossy().into_owned()) -} - #[cfg(test)] -mod android_ndk_resolution_tests { +pub mod tests { + use std::env; + use std::ffi::OsString; + use std::fs; + use std::os::unix::ffi::OsStringExt; + use std::path::Path; use std::path::PathBuf; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::process; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + use enumflags2::BitFlags; - use super::AndroidArchitecture; use super::AndroidNdk; - use super::AndroidNdkDetectionError; use super::find_clang_builtin_includes; use super::validate_ndk_installation; + use crate::build_environment::BuildEnvironment; + use crate::build_environment::tests::environment_of; + use crate::build_environment::tests::variable; + use crate::host_os::HostOs; + + pub const FIXTURE_ANDROID_NDK: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/android_ndk"); + + const FIXTURE_TOOLCHAIN: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/fixtures/android_ndk/toolchains/llvm/prebuilt/linux-x86_64" + ); static NEXT_DIRECTORY_ID: AtomicUsize = AtomicUsize::new(0); fn temporary_directory(name: &str) -> PathBuf { let id = NEXT_DIRECTORY_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "llama-cpp-bindings-{name}-{}-{id}", - std::process::id() - )) + + env::temp_dir().join(format!("llama-cpp-bindings-{name}-{}-{id}", process::id())) } - #[test] - fn every_supported_android_architecture_maps_to_its_abi_and_ndk_prefix() { - let architectures = [ - ("aarch64", "arm64-v8a", "aarch64-linux-android"), - ("arm", "armeabi-v7a", "arm-linux-androideabi"), - ("x86_64", "x86_64", "x86_64-linux-android"), - ("x86", "x86", "i686-linux-android"), - ]; - - for (cargo_cfg_target_arch, abi, prefix) in architectures { - let architecture = AndroidArchitecture::from_cargo_cfg(cargo_cfg_target_arch) - .expect("the architecture is supported"); - - assert_eq!(architecture.abi(), abi); - assert_eq!(architecture.ndk_prefix(), prefix); - } + fn ndk_at(root: &Path) -> BuildEnvironment { + environment_of( + BitFlags::empty(), + &[variable("ANDROID_NDK_HOME", OsString::from(root))], + ) + } + + fn detection_failure( + environment: &BuildEnvironment, + cargo_cfg_target_arch: &str, + host_os: HostOs, + ) -> Option { + AndroidNdk::detect( + environment, + "aarch64-linux-android", + cargo_cfg_target_arch, + host_os, + ) + .err() + .map(|error| error.to_string()) } #[test] - fn an_unsupported_android_architecture_preserves_what_cargo_reported() { - let error = AndroidArchitecture::from_cargo_cfg("riscv64") - .expect_err("riscv64 has no NDK toolchain in this build"); - - assert!(matches!( - error, - AndroidNdkDetectionError::UnsupportedAndroidTarget { cargo_cfg_target_arch } - if cargo_cfg_target_arch == "riscv64" - )); + fn an_installed_ndk_is_detected_with_the_platform_api_level() { + let detected = AndroidNdk::detect( + &environment_of( + BitFlags::empty(), + &[ + variable("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + variable("ANDROID_PLATFORM", OsString::from("android-30")), + ], + ), + "aarch64-linux-android", + "aarch64", + HostOs::Linux, + ) + .unwrap(); + + assert_eq!( + detected, + AndroidNdk { + ndk_path: FIXTURE_ANDROID_NDK.to_owned(), + api_level: "30".to_owned(), + abi: "arm64-v8a", + sysroot: format!("{FIXTURE_TOOLCHAIN}/sysroot"), + target_prefix: "aarch64-linux-android", + clang_builtin_includes: format!("{FIXTURE_TOOLCHAIN}/lib/clang/20/include"), + } + ); } #[test] - fn android_ndk_paths_derive_from_the_resolved_root_and_api_level() { - let ndk = AndroidNdk { - ndk_path: "/opt/android-ndk".to_owned(), - api_level: "35".to_owned(), + fn the_cmake_platform_and_toolchain_file_derive_from_the_api_level_and_ndk_path() { + let android_ndk = AndroidNdk { + ndk_path: "/ndk".to_owned(), + api_level: "30".to_owned(), abi: "arm64-v8a", - host_tag: "linux-x86_64", - toolchain_path: "/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64".to_owned(), - sysroot: "/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot".to_owned(), + sysroot: "/ndk/sysroot".to_owned(), target_prefix: "aarch64-linux-android", - clang_builtin_includes: "/opt/android-ndk/lib/clang/20/include".to_owned(), + clang_builtin_includes: "/ndk/lib/clang/20/include".to_owned(), }; - assert_eq!(ndk.android_platform(), "android-35"); assert_eq!( - ndk.cmake_toolchain_file(), - "/opt/android-ndk/build/cmake/android.toolchain.cmake" + [ + android_ndk.android_platform(), + android_ndk.cmake_toolchain_file() + ], + [ + "android-30".to_owned(), + "/ndk/build/cmake/android.toolchain.cmake".to_owned(), + ] ); } #[test] - fn ndk_validation_rejects_a_missing_root_directory() { - let root = temporary_directory("missing-ndk-root"); - - let error = validate_ndk_installation(root.to_str().expect("UTF-8 temporary path")) - .expect_err("missing NDK root must fail"); + fn an_unprefixed_api_level_is_read_as_is_and_a_missing_one_defaults_to_28() { + let api_level_of = |environment: &BuildEnvironment| { + AndroidNdk::detect( + environment, + "aarch64-linux-android", + "aarch64", + HostOs::Linux, + ) + .map(|ndk| ndk.api_level) + .ok() + }; - assert!(matches!( - error, - AndroidNdkDetectionError::NdkRootMissing { path } if path == root - )); + assert_eq!( + [ + api_level_of(&environment_of( + BitFlags::empty(), + &[ + variable("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + variable("ANDROID_PLATFORM", OsString::from("33")), + ] + )), + api_level_of(&ndk_at(Path::new(FIXTURE_ANDROID_NDK))), + ], + [Some("33".to_owned()), Some("28".to_owned())] + ); } #[test] - fn ndk_validation_rejects_a_root_without_a_cmake_toolchain() { - let root = temporary_directory("ndk-without-toolchain"); - std::fs::create_dir_all(&root).expect("temporary NDK root must be created"); - - let error = validate_ndk_installation(root.to_str().expect("UTF-8 temporary path")) - .expect_err("missing toolchain file must fail"); + fn detection_names_the_ndk_input_that_is_missing_unreadable_or_unsupported() { + let fixture_ndk = Path::new(FIXTURE_ANDROID_NDK); + let missing_root = temporary_directory("missing-ndk-root"); - assert!(matches!( - error, - AndroidNdkDetectionError::NdkToolchainFileMissing { path } - if path == root.join("build/cmake/android.toolchain.cmake") - )); - std::fs::remove_dir_all(root).expect("temporary NDK root must be removed"); + assert_eq!( + [ + detection_failure( + &environment_of(BitFlags::empty(), &[]), + "aarch64", + HostOs::Linux + ), + detection_failure(&ndk_at(&missing_root), "aarch64", HostOs::Linux), + detection_failure(&ndk_at(fixture_ndk), "riscv64", HostOs::Linux), + detection_failure( + &environment_of( + BitFlags::empty(), + &[ + variable("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + variable("ANDROID_PLATFORM", OsString::from_vec(vec![0xff])), + ] + ), + "aarch64", + HostOs::Linux + ), + detection_failure(&ndk_at(fixture_ndk), "aarch64", HostOs::Unsupported), + detection_failure(&ndk_at(fixture_ndk), "aarch64", HostOs::MacOs), + ], + [ + Some( + "Android NDK not found for target aarch64-linux-android. Set ANDROID_NDK_HOME." + .to_owned() + ), + Some(format!( + "Android NDK path does not exist: {}", + missing_root.display() + )), + Some("Unsupported Android target architecture: riscv64".to_owned()), + Some( + "ANDROID_PLATFORM is set but could not be read: environment variable was not \ + valid unicode: \"\\xFF\"" + .to_owned() + ), + Some("Unsupported host platform for Android NDK".to_owned()), + Some(format!( + "Android NDK toolchain not found at: \ + {FIXTURE_ANDROID_NDK}/toolchains/llvm/prebuilt/darwin-x86_64" + )), + ] + ); } #[test] - fn ndk_validation_accepts_a_root_with_a_cmake_toolchain() { - let root = temporary_directory("valid-ndk-root"); - let toolchain = root.join("build/cmake/android.toolchain.cmake"); - std::fs::create_dir_all(toolchain.parent().expect("toolchain parent")) - .expect("toolchain directory must be created"); - std::fs::write(&toolchain, "").expect("toolchain file must be created"); - - validate_ndk_installation(root.to_str().expect("UTF-8 temporary path")) - .expect("valid NDK root must pass"); + fn ndk_validation_rejects_a_root_without_a_cmake_toolchain() { + let root = temporary_directory("ndk-without-toolchain"); + fs::create_dir_all(&root).unwrap(); - std::fs::remove_dir_all(root).expect("temporary NDK root must be removed"); + assert_eq!( + validate_ndk_installation(root.to_str().unwrap()) + .err() + .map(|error| error.to_string()), + Some(format!( + "Android NDK toolchain file not found: {}", + root.join("build/cmake/android.toolchain.cmake").display() + )) + ); + fs::remove_dir_all(root).unwrap(); } #[test] - fn clang_builtin_include_resolution_rejects_a_missing_clang_directory() { - let toolchain = temporary_directory("missing-clang-directory"); - - let error = find_clang_builtin_includes(toolchain.to_str().expect("UTF-8 temporary path")) - .expect_err("missing Clang directory must fail"); + fn detection_rejects_a_toolchain_without_a_clang_directory() { + let root = temporary_directory("ndk-without-clang"); + let toolchain = root.join("toolchains/llvm/prebuilt/linux-x86_64"); + fs::create_dir_all(root.join("build/cmake")).unwrap(); + fs::write(root.join("build/cmake/android.toolchain.cmake"), "").unwrap(); + fs::create_dir_all(&toolchain).unwrap(); - assert!(matches!( - error, - AndroidNdkDetectionError::ClangDirectoryUnreadable { path, .. } - if path == toolchain.join("lib/clang") - )); + assert_eq!( + detection_failure(&ndk_at(&root), "aarch64", HostOs::Linux), + Some(format!( + "Android NDK Clang directory could not be listed at {}: No such file or directory \ + (os error 2)", + toolchain.join("lib/clang").display() + )) + ); + fs::remove_dir_all(root).unwrap(); } #[test] fn clang_builtin_include_resolution_requires_a_version_directory() { let toolchain = temporary_directory("clang-without-version"); - std::fs::create_dir_all(toolchain.join("lib/clang/not-a-version")) - .expect("Clang directory must be created"); - - let error = find_clang_builtin_includes(toolchain.to_str().expect("UTF-8 temporary path")) - .expect_err("missing version directory must fail"); - - assert!(matches!( - error, - AndroidNdkDetectionError::ClangVersionDirectoryMissing { path } - if path == toolchain.join("lib/clang") - )); - std::fs::remove_dir_all(toolchain).expect("temporary toolchain must be removed"); + fs::create_dir_all(toolchain.join("lib/clang")).unwrap(); + fs::write(toolchain.join("lib/clang/21"), "").unwrap(); + + assert_eq!( + find_clang_builtin_includes(toolchain.to_str().unwrap()) + .err() + .map(|error| error.to_string()), + Some(format!( + "Android NDK Clang version directory was not found at {}", + toolchain.join("lib/clang").display() + )) + ); + fs::remove_dir_all(toolchain).unwrap(); } #[test] - fn clang_builtin_include_resolution_requires_an_include_directory() { - let toolchain = temporary_directory("clang-without-includes"); - std::fs::create_dir_all(toolchain.join("lib/clang/20")) - .expect("Clang version directory must be created"); - - let error = find_clang_builtin_includes(toolchain.to_str().expect("UTF-8 temporary path")) - .expect_err("missing built-in includes must fail"); - - assert!(matches!( - error, - AndroidNdkDetectionError::ClangBuiltinIncludesMissing { path } - if path == toolchain.join("lib/clang/20/include") - )); - std::fs::remove_dir_all(toolchain).expect("temporary toolchain must be removed"); + fn clang_builtin_include_resolution_refuses_to_choose_between_version_directories() { + let toolchain = temporary_directory("clang-with-two-versions"); + fs::create_dir_all(toolchain.join("lib/clang/18/include")).unwrap(); + fs::create_dir_all(toolchain.join("lib/clang/17/include")).unwrap(); + + assert_eq!( + find_clang_builtin_includes(toolchain.to_str().unwrap()) + .err() + .map(|error| error.to_string()), + Some(format!( + "Android NDK Clang directory {} holds more than one version directory: {:?}", + toolchain.join("lib/clang").display(), + [ + toolchain.join("lib/clang/17"), + toolchain.join("lib/clang/18") + ] + )) + ); + fs::remove_dir_all(toolchain).unwrap(); } #[test] - fn clang_builtin_include_resolution_returns_the_version_include_directory() { - let toolchain = temporary_directory("clang-with-includes"); - let include = toolchain.join("lib/clang/20/include"); - std::fs::create_dir_all(&include).expect("Clang include directory must be created"); - - let resolved = - find_clang_builtin_includes(toolchain.to_str().expect("UTF-8 temporary path")) - .expect("built-in includes must resolve"); + fn clang_builtin_include_resolution_requires_an_include_directory() { + let toolchain = temporary_directory("clang-without-includes"); + fs::create_dir_all(toolchain.join("lib/clang/20")).unwrap(); - assert_eq!(PathBuf::from(resolved), include); - std::fs::remove_dir_all(toolchain).expect("temporary toolchain must be removed"); + assert_eq!( + find_clang_builtin_includes(toolchain.to_str().unwrap()) + .err() + .map(|error| error.to_string()), + Some(format!( + "Android NDK Clang built-in include directory was not found at {}", + toolchain.join("lib/clang/20/include").display() + )) + ); + fs::remove_dir_all(toolchain).unwrap(); } } diff --git a/llama-cpp-bindings-build/src/apple_variant.rs b/llama-cpp-bindings-build/src/apple_variant.rs deleted file mode 100644 index 2996a38fb..000000000 --- a/llama-cpp-bindings-build/src/apple_variant.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum AppleVariant { - MacOS, - Other, -} diff --git a/llama-cpp-bindings-build/src/bindgen_config.rs b/llama-cpp-bindings-build/src/bindgen_config.rs deleted file mode 100644 index bb899a456..000000000 --- a/llama-cpp-bindings-build/src/bindgen_config.rs +++ /dev/null @@ -1,227 +0,0 @@ -use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; - -use crate::BuildError; -use crate::android_ndk::AndroidNdk; -use crate::debug_log; -use crate::target_os::TargetOs; - -const DEPRECATED_FUNCTIONS: &[&str] = &[ - "ggml_add1", - "ggml_add1_inplace", - "ggml_rope_custom", - "ggml_rope_custom_inplace", - "ggml_type_sizef", - "ggml_upscale_ext", - "llama_add_bos_token", - "llama_add_eos_token", - "llama_copy_state_data", - "llama_free_model", - "llama_get_state_size", - "llama_load_model_from_file", - "llama_load_session_file", - "llama_n_ctx_train", - "llama_n_embd", - "llama_n_head", - "llama_n_layer", - "llama_n_vocab", - "llama_new_context_with_model", - "llama_sampler_init_grammar_lazy", - "llama_save_session_file", - "llama_set_state_data", - "llama_set_warmup", - "llama_token_bos", - "llama_token_cls", - "llama_token_eos", - "llama_token_eot", - "llama_token_fim_mid", - "llama_token_fim_pad", - "llama_token_fim_pre", - "llama_token_fim_rep", - "llama_token_fim_sep", - "llama_token_fim_suf", - "llama_token_get_attr", - "llama_token_get_score", - "llama_token_get_text", - "llama_token_is_control", - "llama_token_is_eog", - "llama_token_nl", - "llama_token_pad", - "llama_token_sep", - "llama_vocab_cls", - "mtmd_encode", - "mtmd_image_tokens_get_nx", - "mtmd_image_tokens_get_ny", -]; - -struct PrivatizedField { - type_name: &'static str, - field_name: &'static str, -} - -const PRIVATIZED_FIELDS: &[PrivatizedField] = &[ - PrivatizedField { - type_name: "llama_context_params", - field_name: "defrag_thold", - }, - PrivatizedField { - type_name: "mtmd_context_params", - field_name: "image_marker", - }, -]; - -#[derive(Clone, Debug)] -struct BindingCallbacks { - privatized_field_hits: Arc>, -} - -impl BindingCallbacks { - fn new() -> Self { - Self { - privatized_field_hits: Arc::new( - PRIVATIZED_FIELDS - .iter() - .map(|_| AtomicBool::new(false)) - .collect(), - ), - } - } - - fn verify_every_privatized_field_was_found(&self) -> Result<(), BuildError> { - for (field, was_found) in PRIVATIZED_FIELDS - .iter() - .zip(self.privatized_field_hits.iter()) - { - if !was_found.load(Ordering::Relaxed) { - return Err(BuildError::PrivatizedFieldMissing { - type_name: field.type_name, - field_name: field.field_name, - }); - } - } - - Ok(()) - } -} - -impl bindgen::callbacks::ParseCallbacks for BindingCallbacks { - fn header_file(&self, filename: &str) { - println!("cargo:rerun-if-changed={filename}"); - } - - fn include_file(&self, filename: &str) { - println!("cargo:rerun-if-changed={filename}"); - } - - fn read_env_var(&self, key: &str) { - println!("cargo:rerun-if-env-changed={key}"); - } - - fn field_visibility( - &self, - info: bindgen::callbacks::FieldInfo<'_>, - ) -> Option { - for (field, was_found) in PRIVATIZED_FIELDS - .iter() - .zip(self.privatized_field_hits.iter()) - { - if field.type_name == info.type_name && field.field_name == info.field_name { - was_found.store(true, Ordering::Relaxed); - - return Some(bindgen::FieldVisibilityKind::Private); - } - } - - None - } -} - -pub fn generate_bindings( - llama_src: &Path, - out_dir: &Path, - target_os: TargetOs, - target_triple: &str, - android_ndk: Option<&AndroidNdk>, -) -> Result<(), BuildError> { - let callbacks = BindingCallbacks::new(); - let mut builder = create_base_builder(llama_src, callbacks.clone()); - - if target_os.is_android() - && let Some(ndk) = android_ndk - { - builder = configure_android_bindgen(builder, ndk, target_triple); - } - - let bindings = builder.generate().map_err(BuildError::Bindgen)?; - - callbacks.verify_every_privatized_field_was_found()?; - - bindings - .write_to_file(out_dir.join("bindings.rs")) - .map_err(BuildError::BindingsWrite)?; - - debug_log!("Bindings Created"); - - Ok(()) -} - -fn create_base_builder(llama_src: &Path, callbacks: BindingCallbacks) -> bindgen::Builder { - let mut builder = bindgen::Builder::default() - .header("wrapper.h") - .clang_arg(format!("-I{}", llama_src.join("include").display())) - .clang_arg(format!("-I{}", llama_src.join("ggml/include").display())) - .parse_callbacks(Box::new(callbacks)) - .derive_partialeq(true) - .allowlist_function("ggml_.*") - .allowlist_type("ggml_.*") - .allowlist_var("GGML_MAX_DIMS") - .allowlist_function("gguf_.*") - .allowlist_type("gguf_.*") - .allowlist_function("llama_.*") - .allowlist_type("llama_.*") - .allowlist_var("LLAMA_TOKEN_NULL") - .allowlist_function("mtmd_.*") - .allowlist_type("mtmd_.*") - .blocklist_function("ggml_fopen") - .blocklist_function("gguf_init_from_file_ptr") - .blocklist_function("gguf_write_to_file_ptr") - .blocklist_function("llama_model_load_from_file_ptr") - .blocklist_type("FILE") - .blocklist_type("_IO_.*") - .prepend_enum_name(false); - - for function in DEPRECATED_FUNCTIONS { - builder = builder.blocklist_function(function); - } - - builder -} - -fn configure_android_bindgen( - mut builder: bindgen::Builder, - ndk: &AndroidNdk, - target_triple: &str, -) -> bindgen::Builder { - builder = builder - .clang_arg(format!("--sysroot={}", ndk.sysroot)) - .clang_arg(format!("-D__ANDROID_API__={}", ndk.api_level)) - .clang_arg("-D__ANDROID__"); - - builder = builder - .clang_arg("-isystem") - .clang_arg(&ndk.clang_builtin_includes); - - builder = builder - .clang_arg("-isystem") - .clang_arg(format!("{}/usr/include/{}", ndk.sysroot, ndk.target_prefix)) - .clang_arg("-isystem") - .clang_arg(format!("{}/usr/include", ndk.sysroot)) - .clang_arg("-include") - .clang_arg("stdbool.h") - .clang_arg("-include") - .clang_arg("stdint.h"); - - builder.clang_arg(format!("--target={target_triple}")) -} diff --git a/llama-cpp-bindings-build/src/binding_callbacks.rs b/llama-cpp-bindings-build/src/binding_callbacks.rs new file mode 100644 index 000000000..c4cec90f5 --- /dev/null +++ b/llama-cpp-bindings-build/src/binding_callbacks.rs @@ -0,0 +1,165 @@ +use std::cell::Cell; +use std::cell::RefCell; +use std::path::PathBuf; +use std::rc::Rc; + +use bindgen::FieldVisibilityKind; +use bindgen::callbacks::DeriveInfo; +use bindgen::callbacks::EnumVariantCustomBehavior; +use bindgen::callbacks::EnumVariantValue; +use bindgen::callbacks::FieldInfo; +use bindgen::callbacks::ParseCallbacks; + +use crate::build_error::BuildError; +use crate::cargo_directive::CargoDirective; +use crate::privatized_field::PrivatizedField; + +const COMPARED_STRUCTS_WITH_FLOATS: &[&str] = &["llama_logit_bias", "llama_token_data"]; + +const COMPARED_STRUCTS_WITHOUT_FLOATS: &[&str] = &["llama_rs_batch_entry"]; + +const SUPERSEDED_ENUM_VARIANTS: &[&str] = &["GGML_PREC_DEFAULT"]; + +fn derives_for(type_name: &str) -> Vec { + if COMPARED_STRUCTS_WITHOUT_FLOATS.contains(&type_name) { + vec!["PartialEq".to_owned(), "Eq".to_owned()] + } else if COMPARED_STRUCTS_WITH_FLOATS.contains(&type_name) { + vec!["PartialEq".to_owned()] + } else { + Vec::new() + } +} + +fn variant_behavior_for(variant_name: &str) -> Option { + SUPERSEDED_ENUM_VARIANTS + .contains(&variant_name) + .then_some(EnumVariantCustomBehavior::Hide) +} + +/// Observes bindgen while it parses: keeps the [`PrivatizedField::ALL`] fields private, hides +/// enum variants llama.cpp superseded, derives the equality the fields of each struct the bindings +/// compare allow, and records what the bindings depend on. +#[derive(Clone, Debug)] +pub struct BindingCallbacks { + privatized_field_hits: Rc<[Cell]>, + rebuild_triggers: Rc>>, +} + +impl Default for BindingCallbacks { + fn default() -> Self { + Self { + privatized_field_hits: PrivatizedField::ALL + .iter() + .map(|_| Cell::new(false)) + .collect(), + rebuild_triggers: Rc::default(), + } + } +} + +impl BindingCallbacks { + /// # Errors + /// + /// Returns [`BuildError::PrivatizedFieldMissing`] for the first privatized field bindgen + /// never saw. + pub fn verify_every_privatized_field_was_found(&self) -> Result<(), BuildError> { + PrivatizedField::ALL + .iter() + .zip(self.privatized_field_hits.iter()) + .find(|(_, was_found)| !was_found.get()) + .map_or(Ok(()), |(field, _)| { + Err(BuildError::PrivatizedFieldMissing { + type_name: field.type_name, + field_name: field.field_name, + }) + }) + } + + /// The headers and environment variables bindgen read, as cargo rebuild triggers. + #[must_use] + pub fn take_rebuild_triggers(&self) -> Vec { + self.rebuild_triggers.take() + } +} + +impl ParseCallbacks for BindingCallbacks { + fn header_file(&self, filename: &str) { + self.rebuild_triggers + .borrow_mut() + .push(CargoDirective::RerunIfChanged(PathBuf::from(filename))); + } + + fn include_file(&self, filename: &str) { + self.rebuild_triggers + .borrow_mut() + .push(CargoDirective::RerunIfChanged(PathBuf::from(filename))); + } + + fn read_env_var(&self, key: &str) { + self.rebuild_triggers + .borrow_mut() + .push(CargoDirective::RerunIfEnvChanged(key.to_owned())); + } + + fn add_derives(&self, info: &DeriveInfo<'_>) -> Vec { + derives_for(info.name) + } + + fn enum_variant_behavior( + &self, + _enum_name: Option<&str>, + original_variant_name: &str, + _variant_value: EnumVariantValue, + ) -> Option { + variant_behavior_for(original_variant_name) + } + + fn field_visibility(&self, info: FieldInfo<'_>) -> Option { + PrivatizedField::ALL + .iter() + .zip(self.privatized_field_hits.iter()) + .find(|(field, _)| { + field.type_name == info.type_name && field.field_name == info.field_name + }) + .map(|(_, was_found)| { + was_found.set(true); + + FieldVisibilityKind::Private + }) + } +} + +#[cfg(test)] +mod tests { + use bindgen::callbacks::EnumVariantCustomBehavior; + + use super::derives_for; + use super::variant_behavior_for; + + #[test] + fn only_superseded_enum_variants_are_hidden() { + assert_eq!( + ["GGML_PREC_DEFAULT", "GGML_PREC_UNDEFINED"].map(variant_behavior_for), + [Some(EnumVariantCustomBehavior::Hide), None] + ); + } + + #[test] + fn the_structs_the_bindings_compare_derive_equality_their_fields_allow() { + assert_eq!( + [ + "llama_token_data", + "llama_logit_bias", + "llama_rs_batch_entry", + "llama_sampler_i" + ] + .map(derives_for), + [ + vec!["PartialEq".to_owned()], + vec!["PartialEq".to_owned()], + vec!["PartialEq".to_owned(), "Eq".to_owned()], + Vec::new() + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/bindings_generation.rs b/llama-cpp-bindings-build/src/bindings_generation.rs new file mode 100644 index 000000000..8c8d7565d --- /dev/null +++ b/llama-cpp-bindings-build/src/bindings_generation.rs @@ -0,0 +1,376 @@ +use std::path::PathBuf; + +use bindgen::Builder; + +use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; + +use crate::binding_callbacks::BindingCallbacks; +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cargo_directive::CargoDirective; +use crate::target_os::TargetOs; + +const DEPRECATED_FUNCTIONS: &[&str] = &[ + "ggml_add1", + "ggml_add1_inplace", + "ggml_flash_attn_ext_set_prec", + "ggml_mul_mat_set_prec", + "ggml_rope_custom", + "ggml_rope_custom_inplace", + "ggml_type_sizef", + "ggml_upscale_ext", + "llama_add_bos_token", + "llama_add_eos_token", + "llama_copy_state_data", + "llama_free_model", + "llama_get_state_size", + "llama_load_model_from_file", + "llama_load_session_file", + "llama_n_ctx_train", + "llama_n_embd", + "llama_n_head", + "llama_n_layer", + "llama_n_vocab", + "llama_new_context_with_model", + "llama_sampler_init_grammar_lazy", + "llama_save_session_file", + "llama_set_state_data", + "llama_set_warmup", + "llama_token_bos", + "llama_token_cls", + "llama_token_eos", + "llama_token_eot", + "llama_token_fim_mid", + "llama_token_fim_pad", + "llama_token_fim_pre", + "llama_token_fim_rep", + "llama_token_fim_sep", + "llama_token_fim_suf", + "llama_token_get_attr", + "llama_token_get_score", + "llama_token_get_text", + "llama_token_is_control", + "llama_token_is_eog", + "llama_token_nl", + "llama_token_pad", + "llama_token_sep", + "llama_vocab_cls", + "mtmd_encode", + "mtmd_helper_model_can_chat", + "mtmd_helper_support_video", + "mtmd_image_tokens_get_nx", + "mtmd_image_tokens_get_ny", +]; + +const SUPERSEDED_ITEMS: &[&str] = &[ + "LLAMA_TOKEN_TYPE_BYTE", + "LLAMA_TOKEN_TYPE_CONTROL", + "LLAMA_TOKEN_TYPE_NORMAL", + "LLAMA_TOKEN_TYPE_UNDEFINED", + "LLAMA_TOKEN_TYPE_UNKNOWN", + "LLAMA_TOKEN_TYPE_UNUSED", + "LLAMA_TOKEN_TYPE_USER_DEFINED", + "ggml_backend_offload_op", + "ggml_backend_supports_buft", + "ggml_backend_supports_op", + "llama_batch", + "llama_batch_free", + "llama_batch_get_one", + "llama_batch_init", + "llama_decode", + "llama_encode", + "llama_get_embeddings", + "llama_get_logits", + "llama_perf_context", + "llama_perf_context_data", + "llama_perf_context_print", + "llama_perf_context_reset", + "llama_perf_sampler", + "llama_perf_sampler_data", + "llama_perf_sampler_print", + "llama_perf_sampler_reset", + "llama_token_type", + "mtmd_image_tokens_get_id", + "mtmd_image_tokens_get_n_pos", + "mtmd_image_tokens_get_n_tokens", +]; + +fn configured_builder() -> Builder { + let builder = Builder::default() + .generate_comments(false) + .allowlist_function("ggml_.*") + .allowlist_type("ggml_.*") + .allowlist_var("GGML_MAX_DIMS") + .allowlist_function("gguf_.*") + .allowlist_type("gguf_.*") + .allowlist_function("llama_.*") + .allowlist_type("llama_.*") + .allowlist_var("LLAMA_TOKEN_NULL") + .allowlist_var("LLAMA_DRY_MAX_SEQ_BREAKERS") + .allowlist_function("mtmd_.*") + .allowlist_type("mtmd_.*") + .blocklist_function("ggml_fopen") + .blocklist_function("llama_adapter_lora_init_from_file_ptr") + .blocklist_function("gguf_init_from_file_ptr") + .blocklist_function("gguf_write_to_file_ptr") + .blocklist_function("llama_model_load_from_file_ptr") + .blocklist_type("FILE") + .blocklist_type("_IO_.*") + .prepend_enum_name(false); + + let builder = DEPRECATED_FUNCTIONS + .iter() + .fold(builder, Builder::blocklist_function); + + SUPERSEDED_ITEMS + .iter() + .fold(builder, Builder::blocklist_item) +} + +fn fault_injection_args(environment: &BuildEnvironment) -> Vec { + if environment.has(BuildFeature::FaultInjection) { + vec![format!("-D{FAULT_INJECTION_DEFINE}")] + } else { + Vec::new() + } +} + +fn target_args(context: &BuildContext) -> Vec { + match &context.target_os { + TargetOs::Android(android_ndk) => android_ndk.clang_args(&context.target_triple), + TargetOs::Apple | TargetOs::Linux => Vec::new(), + } +} + +/// Generates the Rust bindings of `header` with bindgen into `output`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BindingsGeneration { + pub header: PathBuf, + pub clang_args: Vec, + pub output: PathBuf, +} + +impl BindingsGeneration { + #[must_use] + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Self { + let llama_src = context.llama_src(); + + Self { + header: context.sys_dir.join("wrapper.h"), + clang_args: [ + vec![ + format!("-I{}", llama_src.display()), + format!("-I{}", llama_src.join("include").display()), + format!("-I{}", llama_src.join("ggml/include").display()), + ], + fault_injection_args(environment), + target_args(context), + ] + .concat(), + output: context.out_dir.join("bindings.rs"), + } + } + + /// Writes the bindings and returns the cargo directives that rebuild them when what they + /// were generated from changes. + /// + /// # Errors + /// + /// Returns [`BuildError::Bindgen`] when bindgen cannot parse the header, + /// [`BuildError::PrivatizedFieldMissing`] when a field the bindings keep private no longer + /// exists, or [`BuildError::BindingsWrite`] when `output` cannot be written. + pub fn generate(&self) -> Result, BuildError> { + let callbacks = BindingCallbacks::default(); + let bindings = configured_builder() + .header(self.header.display().to_string()) + .clang_args(&self.clang_args) + .parse_callbacks(Box::new(callbacks.clone())) + .generate() + .map_err(BuildError::Bindgen)?; + + callbacks.verify_every_privatized_field_was_found()?; + bindings + .write_to_file(&self.output) + .map_err(BuildError::BindingsWrite)?; + + Ok([ + callbacks.take_rebuild_triggers(), + vec![CargoDirective::Debug("Bindings Created".to_owned())], + ] + .concat()) + } +} + +#[cfg(test)] +mod tests { + use std::env; + use std::fs; + use std::path::PathBuf; + use std::process; + + use enumflags2::BitFlags; + + use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; + + use super::BindingsGeneration; + use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; + use crate::build_context::tests::android_overrides; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_environment::tests::EnvironmentVariable; + use crate::build_feature::BuildFeature; + + const FIXTURE_BINDINGS: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/bindings"); + + fn planned( + features: BitFlags, + overrides: &[EnvironmentVariable], + ) -> BindingsGeneration { + let environment = cargo_environment(features, overrides); + + BindingsGeneration::of(&context_of(&environment), &environment) + } + + fn llama_cpp_includes() -> Vec { + vec![ + "-I/sys/llama.cpp".to_owned(), + "-I/sys/llama.cpp/include".to_owned(), + "-I/sys/llama.cpp/ggml/include".to_owned(), + ] + } + + fn generation_failure(header: &str, output: PathBuf) -> Option { + BindingsGeneration { + header: PathBuf::from(FIXTURE_BINDINGS).join(header), + clang_args: Vec::new(), + output, + } + .generate() + .err() + .map(|error| error.to_string()) + } + + #[test] + fn the_wrapper_header_is_parsed_against_the_llama_cpp_headers() { + assert_eq!( + planned(BitFlags::empty(), &[]), + BindingsGeneration { + header: PathBuf::from("/sys/wrapper.h"), + clang_args: llama_cpp_includes(), + output: PathBuf::from("/out/bindings.rs"), + } + ); + } + + #[test] + fn fault_injection_adds_its_define_to_the_clang_arguments() { + assert_eq!( + planned(BitFlags::from(BuildFeature::FaultInjection), &[]).clang_args, + [ + llama_cpp_includes(), + vec![format!("-D{FAULT_INJECTION_DEFINE}")], + ] + .concat() + ); + } + + #[test] + fn android_targets_add_the_ndk_clang_arguments() { + let toolchain = format!("{FIXTURE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64"); + + assert_eq!( + planned(BitFlags::empty(), &android_overrides()).clang_args, + [ + llama_cpp_includes(), + vec![ + format!("--sysroot={toolchain}/sysroot"), + "-D__ANDROID_API__=28".to_owned(), + "-D__ANDROID__".to_owned(), + "-isystem".to_owned(), + format!("{toolchain}/lib/clang/20/include"), + "-isystem".to_owned(), + format!("{toolchain}/sysroot/usr/include/aarch64-linux-android"), + "-isystem".to_owned(), + format!("{toolchain}/sysroot/usr/include"), + "-include".to_owned(), + "stdbool.h".to_owned(), + "-include".to_owned(), + "stdint.h".to_owned(), + "--target=aarch64-linux-android".to_owned(), + ], + ] + .concat() + ); + } + + #[test] + fn a_missing_header_is_reported_by_bindgen() { + assert_eq!( + generation_failure("missing.h", PathBuf::from("/out/bindings.rs")), + Some(format!( + "bindgen failed: header '{FIXTURE_BINDINGS}/missing.h' does not exist." + )) + ); + } + + #[test] + fn a_header_without_a_privatized_field_is_refused() { + assert_eq!( + generation_failure( + "missing_privatized_field.h", + PathBuf::from("/out/bindings.rs") + ), + Some( + "bindgen never saw field llama_context_params::defrag_thold, so it can no longer \ + be kept private" + .to_owned() + ) + ); + } + + #[test] + fn generated_bindings_hide_superseded_enum_variants_and_keep_their_replacements() { + let output = env::temp_dir().join(format!( + "llama-cpp-bindings-build-bindings-{}.rs", + process::id() + )); + BindingsGeneration { + header: PathBuf::from(FIXTURE_BINDINGS).join("wrapper.h"), + clang_args: Vec::new(), + output: output.clone(), + } + .generate() + .unwrap(); + let bindings = fs::read_to_string(&output).unwrap(); + fs::remove_file(&output).unwrap(); + let declares = |constant: &str| { + bindings + .lines() + .any(|line| line.trim() == format!("pub const {constant}: ggml_prec = 0;")) + }; + + assert_eq!( + [ + declares("GGML_PREC_DEFAULT"), + declares("GGML_PREC_UNDEFINED") + ], + [false, true] + ); + } + + #[test] + fn bindings_that_cannot_be_written_are_reported() { + assert_eq!( + generation_failure( + "wrapper.h", + PathBuf::from(FIXTURE_BINDINGS).join("missing-directory/bindings.rs") + ), + Some( + "generated bindings could not be written: No such file or directory (os error 2)" + .to_owned() + ) + ); + } +} diff --git a/llama-cpp-bindings-build/src/build_context.rs b/llama-cpp-bindings-build/src/build_context.rs new file mode 100644 index 000000000..b8261434b --- /dev/null +++ b/llama-cpp-bindings-build/src/build_context.rs @@ -0,0 +1,213 @@ +use std::path::PathBuf; + +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cargo_directive::CargoDirective; +use crate::host_os::HostOs; +use crate::target_os::TargetOs; + +/// Where and for what the native libraries are built, as cargo describes the build script run. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BuildContext { + pub out_dir: PathBuf, + pub sys_dir: PathBuf, + pub target_os: TargetOs, + pub target_triple: String, + pub host_triple: String, + pub cargo_cfg_target_arch: String, + pub build_shared_libs: bool, + pub opt_level: String, + pub debug_info: String, + pub include_debug_messages: bool, +} + +impl BuildContext { + /// # Errors + /// + /// Returns [`BuildError::Environment`] for a variable cargo sets for build scripts that is + /// missing or not Unicode, or the [`BuildError`] of [`TargetOs::detect`] when the target + /// cannot be built for. + pub fn from_environment( + environment: &BuildEnvironment, + host_os: HostOs, + ) -> Result { + let target_triple = environment.required("TARGET")?; + let cargo_cfg_target_arch = environment.required("CARGO_CFG_TARGET_ARCH")?; + + Ok(Self { + out_dir: PathBuf::from(environment.required("OUT_DIR")?), + sys_dir: PathBuf::from(environment.required("CARGO_MANIFEST_DIR")?), + target_os: TargetOs::detect( + environment, + &target_triple, + &cargo_cfg_target_arch, + host_os, + )?, + host_triple: environment.required("HOST")?, + build_shared_libs: environment.has(BuildFeature::DynamicLink), + opt_level: environment.required("OPT_LEVEL")?, + debug_info: environment.required("DEBUG")?, + include_debug_messages: environment.optional("BUILD_DEBUG")?.is_some(), + target_triple, + cargo_cfg_target_arch, + }) + } + + #[must_use] + pub fn cmake_dir(&self) -> PathBuf { + self.out_dir.join("cmake") + } + + #[must_use] + pub fn cmake_build_dir(&self) -> PathBuf { + self.cmake_dir().join("build") + } + + #[must_use] + pub fn llama_src(&self) -> PathBuf { + self.sys_dir.join("llama.cpp") + } + + #[must_use] + pub fn debug_messages(&self) -> Vec { + [ + format!("TARGET: {}", self.target_triple), + format!("CARGO_MANIFEST_DIR: {}", self.sys_dir.display()), + format!("OUT_DIR: {}", self.out_dir.display()), + format!("CMAKE_DIR: {}", self.cmake_dir().display()), + format!("BUILD_SHARED: {}", self.build_shared_libs), + ] + .map(CargoDirective::Debug) + .to_vec() + } +} + +#[cfg(test)] +pub mod tests { + use std::ffi::OsStr; + use std::path::PathBuf; + + use enumflags2::BitFlags; + + use super::BuildContext; + use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; + use crate::build_environment::BuildEnvironment; + use crate::build_environment::tests::EnvironmentVariable; + use crate::build_environment::tests::environment_of; + use crate::build_environment::tests::variable; + use crate::build_environment::tests::with_non_unicode; + use crate::build_feature::BuildFeature; + use crate::cargo_directive::CargoDirective; + use crate::host_os::HostOs; + use crate::target_os::TargetOs; + + fn cargo_variables() -> [EnvironmentVariable; 8] { + [ + variable("CARGO_CFG_TARGET_ARCH", "x86_64"), + variable("CARGO_CFG_TARGET_OS", "linux"), + variable("CARGO_MANIFEST_DIR", "/sys"), + variable("DEBUG", "false"), + variable("HOST", "x86_64-unknown-linux-gnu"), + variable("OPT_LEVEL", "3"), + variable("OUT_DIR", "/out"), + variable("TARGET", "x86_64-unknown-linux-gnu"), + ] + } + + pub fn android_overrides() -> [EnvironmentVariable; 4] { + [ + variable("ANDROID_NDK_HOME", FIXTURE_ANDROID_NDK), + variable("CARGO_CFG_TARGET_ARCH", "aarch64"), + variable("CARGO_CFG_TARGET_OS", "android"), + variable("TARGET", "aarch64-linux-android"), + ] + } + + /// The variables cargo sets for a build script building for x86-64 Linux, with `overrides`. + pub fn cargo_environment( + features: BitFlags, + overrides: &[EnvironmentVariable], + ) -> BuildEnvironment { + environment_of(features, &[&cargo_variables()[..], overrides].concat()) + } + + pub fn context_of(environment: &BuildEnvironment) -> BuildContext { + BuildContext::from_environment(environment, HostOs::Linux).unwrap() + } + + #[test] + fn the_context_is_read_from_the_build_variables_and_the_dynamic_link_feature() { + assert_eq!( + context_of(&cargo_environment( + BitFlags::from(BuildFeature::DynamicLink), + &[variable("BUILD_DEBUG", "")] + )), + BuildContext { + out_dir: PathBuf::from("/out"), + sys_dir: PathBuf::from("/sys"), + target_os: TargetOs::Linux, + target_triple: "x86_64-unknown-linux-gnu".to_owned(), + host_triple: "x86_64-unknown-linux-gnu".to_owned(), + cargo_cfg_target_arch: "x86_64".to_owned(), + build_shared_libs: true, + opt_level: "3".to_owned(), + debug_info: "false".to_owned(), + include_debug_messages: true, + } + ); + } + + #[test] + fn every_variable_cargo_sets_for_build_scripts_is_required() { + let without = |name: &str| { + let mut environment = cargo_environment(BitFlags::empty(), &[]); + + environment.variables.remove(OsStr::new(name)); + + BuildContext::from_environment(&environment, HostOs::Linux) + .err() + .map(|error| error.to_string()) + }; + let required = cargo_variables().map(|EnvironmentVariable { name, .. }| name); + + assert_eq!( + required.map(without), + required.map(|name| Some(format!( + "environment variable {name} could not be read: environment variable not found" + ))) + ); + } + + #[test] + fn a_build_debug_request_that_is_not_unicode_is_refused() { + assert_eq!( + BuildContext::from_environment( + &with_non_unicode(cargo_environment(BitFlags::empty(), &[]), "BUILD_DEBUG"), + HostOs::Linux, + ) + .err() + .map(|error| error.to_string()), + Some( + "environment variable BUILD_DEBUG could not be read: environment variable was \ + not valid unicode: \"\\xFF\"" + .to_owned() + ) + ); + } + + #[test] + fn debug_messages_describe_where_the_build_happens() { + assert_eq!( + context_of(&cargo_environment(BitFlags::empty(), &[])).debug_messages(), + [ + "TARGET: x86_64-unknown-linux-gnu", + "CARGO_MANIFEST_DIR: /sys", + "OUT_DIR: /out", + "CMAKE_DIR: /out/cmake", + "BUILD_SHARED: false", + ] + .map(|message| CargoDirective::Debug(message.to_owned())) + ); + } +} diff --git a/llama-cpp-bindings-build/src/build_environment.rs b/llama-cpp-bindings-build/src/build_environment.rs new file mode 100644 index 000000000..472c8be46 --- /dev/null +++ b/llama-cpp-bindings-build/src/build_environment.rs @@ -0,0 +1,209 @@ +use std::collections::BTreeMap; +use std::env; +use std::env::VarError; +use std::ffi::OsStr; +use std::ffi::OsString; +use std::path::PathBuf; + +use enumflags2::BitFlags; +use find_cuda_helper::find_cuda_lib_dirs; + +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::enabled_build_features::enabled_build_features; + +/// Everything the native build reads from outside: the environment variables, the cargo features +/// this crate was compiled with, and the CUDA library directories installed on the host. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BuildEnvironment { + pub variables: BTreeMap, + pub features: BitFlags, + pub cuda_library_dirs: Vec, +} + +impl BuildEnvironment { + /// The environment of the running build script. + #[must_use] + pub fn of_process() -> Self { + Self { + variables: env::vars_os().collect(), + features: enabled_build_features(), + cuda_library_dirs: find_cuda_lib_dirs(), + } + } + + /// # Errors + /// + /// Returns [`VarError::NotPresent`] for a variable the environment lacks, or + /// [`VarError::NotUnicode`] for one that is not Unicode. + pub fn var(&self, name: &'static str) -> Result { + self.variables + .get(OsStr::new(name)) + .cloned() + .ok_or(VarError::NotPresent)? + .into_string() + .map_err(VarError::NotUnicode) + } + + /// # Errors + /// + /// Returns [`BuildError::Environment`] when the variable is missing or not Unicode. + pub fn required(&self, name: &'static str) -> Result { + self.var(name) + .map_err(|source| BuildError::Environment { name, source }) + } + + /// # Errors + /// + /// Returns [`BuildError::Environment`] when the variable is set but not Unicode. + pub fn optional(&self, name: &'static str) -> Result, BuildError> { + match self.var(name) { + Ok(value) => Ok(Some(value)), + Err(VarError::NotPresent) => Ok(None), + Err(source) => Err(BuildError::Environment { name, source }), + } + } + + #[must_use] + pub fn has(&self, feature: BuildFeature) -> bool { + self.features.contains(feature) + } +} + +#[cfg(test)] +pub mod tests { + use std::collections::BTreeMap; + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + use enumflags2::BitFlags; + use find_cuda_helper::find_cuda_lib_dirs; + + use super::BuildEnvironment; + use crate::build_feature::BuildFeature; + use crate::enabled_build_features::enabled_build_features; + + #[derive(Clone, Debug)] + pub struct EnvironmentVariable { + pub name: &'static str, + pub value: OsString, + } + + #[must_use] + pub fn variable>( + name: &'static str, + value: TValue, + ) -> EnvironmentVariable { + EnvironmentVariable { + name, + value: value.into(), + } + } + + #[must_use] + pub fn environment_of( + features: BitFlags, + variables: &[EnvironmentVariable], + ) -> BuildEnvironment { + let mut environment = BuildEnvironment { + variables: BTreeMap::new(), + features, + cuda_library_dirs: Vec::new(), + }; + + for EnvironmentVariable { name, value } in variables { + environment + .variables + .insert(OsString::from(name), value.clone()); + } + + environment + } + + #[must_use] + pub fn with_non_unicode(mut environment: BuildEnvironment, name: &str) -> BuildEnvironment { + environment + .variables + .insert(OsString::from(name), OsString::from_vec(vec![0xff])); + + environment + } + + #[test] + fn the_process_environment_holds_the_variables_features_and_cuda_lookup_of_the_process() { + let environment = BuildEnvironment::of_process(); + + assert_eq!( + environment.var("CARGO_MANIFEST_DIR"), + Ok(env!("CARGO_MANIFEST_DIR").to_owned()) + ); + assert_eq!(environment.features, enabled_build_features()); + assert_eq!(environment.cuda_library_dirs, find_cuda_lib_dirs()); + } + + #[test] + fn a_unicode_variable_is_read() { + let environment = environment_of( + BitFlags::empty(), + &[variable( + "TARGET", + OsString::from("x86_64-unknown-linux-gnu"), + )], + ); + + assert_eq!( + [ + environment.required("TARGET").ok(), + environment.optional("TARGET").ok().flatten() + ], + [(); 2].map(|()| Some("x86_64-unknown-linux-gnu".to_owned())) + ); + } + + #[test] + fn a_missing_variable_fails_when_required_and_is_none_when_optional() { + let environment = environment_of(BitFlags::empty(), &[]); + + assert_eq!(environment.optional("HOST").ok(), Some(None)); + assert_eq!( + environment + .required("HOST") + .err() + .map(|error| error.to_string()), + Some( + "environment variable HOST could not be read: environment variable not found" + .to_owned() + ) + ); + } + + #[test] + fn a_variable_that_is_not_unicode_is_refused_even_when_optional() { + let environment = with_non_unicode(environment_of(BitFlags::empty(), &[]), "TARGET"); + + assert_eq!( + environment + .optional("TARGET") + .err() + .map(|error| error.to_string()), + Some( + "environment variable TARGET could not be read: environment variable was not \ + valid unicode: \"\\xFF\"" + .to_owned() + ) + ); + } + + #[test] + fn features_are_reported_by_membership() { + let environment = environment_of(BitFlags::from(BuildFeature::Cuda), &[]); + + assert_eq!( + [ + environment.has(BuildFeature::Cuda), + environment.has(BuildFeature::Metal) + ], + [true, false] + ); + } +} diff --git a/llama-cpp-bindings-build/src/build_error.rs b/llama-cpp-bindings-build/src/build_error.rs new file mode 100644 index 000000000..9aa224e80 --- /dev/null +++ b/llama-cpp-bindings-build/src/build_error.rs @@ -0,0 +1,107 @@ +use std::env::VarError; +use std::io; +use std::path::PathBuf; + +#[derive(Debug, thiserror::Error)] +pub enum BuildError { + #[error("environment variable {name} could not be read: {source}")] + Environment { + name: &'static str, + #[source] + source: VarError, + }, + #[error("unsupported target operating system: {cargo_cfg_target_os}")] + UnsupportedTargetOs { cargo_cfg_target_os: String }, + #[error("Android NDK not found for target {target_triple}. Set ANDROID_NDK_HOME.")] + AndroidNdkRootNotConfigured { + target_triple: String, + #[source] + source: VarError, + }, + #[error("Android NDK path does not exist: {path}")] + AndroidNdkRootMissing { path: PathBuf }, + #[error("Android NDK toolchain file not found: {path}")] + AndroidNdkToolchainFileMissing { path: PathBuf }, + #[error("Android NDK toolchain not found at: {path}")] + AndroidNdkToolchainDirectoryMissing { path: PathBuf }, + #[error("Unsupported host platform for Android NDK")] + AndroidNdkUnsupportedHostPlatform, + #[error("Unsupported Android target architecture: {cargo_cfg_target_arch}")] + UnsupportedAndroidTarget { cargo_cfg_target_arch: String }, + #[error("ANDROID_PLATFORM is set but could not be read: {source}")] + AndroidPlatformUnreadable { + #[source] + source: VarError, + }, + #[error("Android NDK Clang directory could not be listed at {path}: {source}")] + AndroidNdkClangDirectoryUnreadable { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("Android NDK Clang version directory was not found at {path}")] + AndroidNdkClangVersionDirectoryMissing { path: PathBuf }, + #[error( + "Android NDK Clang directory {path} holds more than one version directory: {candidates:?}" + )] + AndroidNdkClangVersionDirectoryAmbiguous { + path: PathBuf, + candidates: Vec, + }, + #[error("Android NDK Clang built-in include directory was not found at {path}")] + AndroidNdkClangBuiltinIncludesMissing { path: PathBuf }, + #[error("bindgen failed: {0}")] + Bindgen(#[source] bindgen::BindgenError), + #[error( + "bindgen never saw field {type_name}::{field_name}, so it can no longer be kept private" + )] + PrivatizedFieldMissing { + type_name: &'static str, + field_name: &'static str, + }, + #[error("generated bindings could not be written: {0}")] + BindingsWrite(#[source] io::Error), + #[error("native wrapper compilation failed: {0}")] + NativeWrapper(#[source] cc::Error), + #[error("filesystem operation failed for {path}: {source}")] + Filesystem { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("environment path {name} does not name a directory: {path}")] + EnvironmentDirectory { name: &'static str, path: PathBuf }, + #[error("cargo directives could not be written: {0}")] + DirectiveWrite(#[source] io::Error), + #[error("the CMake file API reply directory {path} could not be listed: {source}")] + CmakeFileApiReplyDirectoryUnreadable { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("CMake wrote no file API reply index into {path}")] + CmakeFileApiReplyIndexMissing { path: PathBuf }, + #[error("the CMake file API reply file {path} could not be read: {source}")] + CmakeFileApiReplyFileUnreadable { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("the CMake file API reply file {path} is not the reply this crate queried: {source}")] + CmakeFileApiReplyFileInvalid { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + #[error("the CMake cache has no entry {name}")] + CmakeCacheEntryMissing { name: String }, + #[error("the CMake cache entry {name} is {value}, neither ON nor OFF")] + CmakeCacheEntryNotASwitch { name: String, value: String }, + #[error( + "ggml needs the OpenMP runtime {libraries:?}, which Apple's linker cannot link by its \ + exact file name" + )] + OpenmpRuntimeUnlinkableOnApple { libraries: Vec }, + #[error("the OpenMP runtime library {library} names no file")] + OpenmpRuntimeLibraryWithoutFileName { library: PathBuf }, +} diff --git a/llama-cpp-bindings-build/src/build_feature.rs b/llama-cpp-bindings-build/src/build_feature.rs new file mode 100644 index 000000000..b690db85a --- /dev/null +++ b/llama-cpp-bindings-build/src/build_feature.rs @@ -0,0 +1,20 @@ +use enumflags2::bitflags; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[bitflags] +#[repr(u16)] +pub enum BuildFeature { + Cuda = 1 << 0, + CudaNoVmm = 1 << 1, + DynamicBackends = 1 << 2, + DynamicLink = 1 << 3, + FaultInjection = 1 << 4, + Metal = 1 << 5, + Openmp = 1 << 6, + Rocm = 1 << 7, + SharedStdcxx = 1 << 8, + StaticStdcxx = 1 << 9, + SystemGgml = 1 << 10, + SystemGgmlStatic = 1 << 11, + Vulkan = 1 << 12, +} diff --git a/llama-cpp-bindings-build/src/cargo_directive.rs b/llama-cpp-bindings-build/src/cargo_directive.rs new file mode 100644 index 000000000..680a19f81 --- /dev/null +++ b/llama-cpp-bindings-build/src/cargo_directive.rs @@ -0,0 +1,100 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; +use std::path::PathBuf; + +use crate::native_library::NativeLibrary; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum CargoDirective { + RerunIfChanged(PathBuf), + RerunIfEnvChanged(String), + LinkSearch(PathBuf), + LinkSearchNative(PathBuf), + LinkLibrary(NativeLibrary), + LinkDylibByFileName(PathBuf), + Debug(String), +} + +impl CargoDirective { + #[must_use] + pub const fn is_debug(&self) -> bool { + matches!(self, Self::Debug(_)) + } +} + +impl Display for CargoDirective { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::RerunIfChanged(path) => { + write!(formatter, "cargo:rerun-if-changed={}", path.display()) + } + Self::RerunIfEnvChanged(name) => write!(formatter, "cargo:rerun-if-env-changed={name}"), + Self::LinkSearch(path) => { + write!(formatter, "cargo:rustc-link-search={}", path.display()) + } + Self::LinkSearchNative(path) => { + write!( + formatter, + "cargo:rustc-link-search=native={}", + path.display() + ) + } + Self::LinkLibrary(library) => write!(formatter, "cargo:rustc-link-lib={library}"), + Self::LinkDylibByFileName(file_name) => write!( + formatter, + "cargo:rustc-link-lib=dylib:+verbatim={}", + file_name.display() + ), + Self::Debug(message) => write!(formatter, "cargo:warning=[DEBUG] {message}"), + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::CargoDirective; + use crate::link_kind::LinkKind; + use crate::native_library::NativeLibrary; + + fn every_directive() -> [CargoDirective; 7] { + [ + CargoDirective::RerunIfChanged(PathBuf::from("wrapper.h")), + CargoDirective::RerunIfEnvChanged("CUDAARCHS".to_owned()), + CargoDirective::LinkSearch(PathBuf::from("/out/lib")), + CargoDirective::LinkSearchNative(PathBuf::from("/out/build/common")), + CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Static, + name: "llama", + }), + CargoDirective::LinkDylibByFileName(PathBuf::from("libgomp.so")), + CargoDirective::Debug("TARGET: x86_64-unknown-linux-gnu".to_owned()), + ] + } + + #[test] + fn every_directive_is_written_in_the_cargo_syntax() { + assert_eq!( + every_directive().map(|directive| directive.to_string()), + [ + "cargo:rerun-if-changed=wrapper.h", + "cargo:rerun-if-env-changed=CUDAARCHS", + "cargo:rustc-link-search=/out/lib", + "cargo:rustc-link-search=native=/out/build/common", + "cargo:rustc-link-lib=static=llama", + "cargo:rustc-link-lib=dylib:+verbatim=libgomp.so", + "cargo:warning=[DEBUG] TARGET: x86_64-unknown-linux-gnu", + ] + ); + } + + #[test] + fn only_debug_messages_are_debug_directives() { + assert_eq!( + every_directive().map(|directive| directive.is_debug()), + [false, false, false, false, false, false, true] + ); + } +} diff --git a/llama-cpp-bindings-build/src/cmake_build.rs b/llama-cpp-bindings-build/src/cmake_build.rs new file mode 100644 index 000000000..a039f9f59 --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_build.rs @@ -0,0 +1,59 @@ +use std::fs; +use std::path::PathBuf; + +use cmake::Config; + +use crate::build_error::BuildError; +use crate::cmake_cache::CmakeCache; +use crate::cmake_define::CmakeDefine; +use crate::cmake_file_api::CmakeFileApi; +use crate::native_profile::NATIVE_PROFILE; + +/// Configures the `CMake` project in `source_dir` and builds and installs it into `out_dir`, +/// reading back through `file_api` the cache `CMake` configured it with. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CmakeBuild { + pub source_dir: PathBuf, + pub out_dir: PathBuf, + pub file_api: CmakeFileApi, + pub target_triple: String, + pub host_triple: String, + pub defines: Vec, + pub required_directories: Vec, +} + +impl CmakeBuild { + /// # Errors + /// + /// Returns [`BuildError::Filesystem`] when a directory the project installs into cannot be + /// created, or the [`BuildError`] of [`CmakeFileApi::cache`]. + /// + /// # Panics + /// + /// Panics when `CMake` fails, the way the `cmake` crate reports a failed build. + pub fn build(&self) -> Result { + self.required_directories.iter().try_for_each(|directory| { + fs::create_dir_all(directory).map_err(|source| BuildError::Filesystem { + path: directory.clone(), + source, + }) + })?; + + let mut config = Config::new(&self.source_dir); + + for CmakeDefine { name, value } in &self.defines { + config.define(name, value); + } + + config + .configure_arg("-C") + .configure_arg(CmakeFileApi::CACHE_QUERY_SCRIPT) + .target(&self.target_triple) + .host(&self.host_triple) + .out_dir(&self.out_dir) + .profile(NATIVE_PROFILE) + .build(); + + self.file_api.cache() + } +} diff --git a/llama-cpp-bindings-build/src/cmake_cache.rs b/llama-cpp-bindings-build/src/cmake_cache.rs new file mode 100644 index 000000000..a126495ad --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_cache.rs @@ -0,0 +1,101 @@ +use serde::Deserialize; + +use crate::build_error::BuildError; +use crate::cmake_cache_entry::CmakeCacheEntry; + +/// The `CMake` cache of a configured build tree, read through the file API's `cache` object. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeCache { + pub entries: Vec, +} + +impl CmakeCache { + #[must_use] + pub fn contains(&self, name: &str) -> bool { + self.entries.iter().any(|entry| entry.name == name) + } + + /// # Errors + /// + /// Returns [`BuildError::CmakeCacheEntryMissing`] when the cache has no entry `name`. + pub fn value(&self, name: &str) -> Result<&str, BuildError> { + self.entries + .iter() + .find(|entry| entry.name == name) + .map(|entry| entry.value.as_str()) + .ok_or_else(|| BuildError::CmakeCacheEntryMissing { + name: name.to_owned(), + }) + } + + /// The switch `name`, which the project writes into the cache as `ON` or `OFF`. + /// + /// # Errors + /// + /// Returns [`BuildError::CmakeCacheEntryMissing`] when the cache has no entry `name`, or + /// [`BuildError::CmakeCacheEntryNotASwitch`] when it holds anything but `ON` or `OFF`. + pub fn switch(&self, name: &str) -> Result { + match self.value(name)? { + "ON" => Ok(true), + "OFF" => Ok(false), + value => Err(BuildError::CmakeCacheEntryNotASwitch { + name: name.to_owned(), + value: value.to_owned(), + }), + } + } + + /// The elements of the `CMake` list `name`, which `CMake` separates with semicolons and + /// writes as an empty value when the list has none. + /// + /// # Errors + /// + /// Returns [`BuildError::CmakeCacheEntryMissing`] when the cache has no entry `name`. + pub fn list(&self, name: &str) -> Result, BuildError> { + Ok(self + .value(name)? + .split(';') + .filter(|element| !element.is_empty()) + .collect()) + } +} + +#[cfg(test)] +pub mod tests { + use super::CmakeCache; + use crate::cmake_cache_entry::CmakeCacheEntry; + + pub fn entry(name: &str, value: &str) -> CmakeCacheEntry { + CmakeCacheEntry { + name: name.to_owned(), + value: value.to_owned(), + } + } + + #[test] + fn a_switch_written_as_anything_but_on_or_off_is_refused() { + assert_eq!( + CmakeCache { + entries: vec![entry("GGML_OPENMP_ENABLED", "TRUE")], + } + .switch("GGML_OPENMP_ENABLED") + .err() + .map(|error| error.to_string()), + Some( + "the CMake cache entry GGML_OPENMP_ENABLED is TRUE, neither ON nor OFF".to_owned() + ) + ); + } + + #[test] + fn an_empty_list_has_no_elements() { + assert_eq!( + CmakeCache { + entries: vec![entry("OpenMP_C_LIB_NAMES", "")], + } + .list("OpenMP_C_LIB_NAMES") + .unwrap(), + Vec::<&str>::new() + ); + } +} diff --git a/llama-cpp-bindings-build/src/cmake_cache_entry.rs b/llama-cpp-bindings-build/src/cmake_cache_entry.rs new file mode 100644 index 000000000..15ecdb9ab --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_cache_entry.rs @@ -0,0 +1,8 @@ +use serde::Deserialize; + +/// One entry of the `CMake` cache, as the file API's `cache` object lists it. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeCacheEntry { + pub name: String, + pub value: String, +} diff --git a/llama-cpp-bindings-build/src/cmake_config.rs b/llama-cpp-bindings-build/src/cmake_config.rs deleted file mode 100644 index 742b05238..000000000 --- a/llama-cpp-bindings-build/src/cmake_config.rs +++ /dev/null @@ -1,297 +0,0 @@ -use std::env; -use std::path::{Path, PathBuf}; - -use cmake::Config; - -use crate::BuildContext; -use crate::BuildError; -use crate::android_ndk::AndroidNdk; -use crate::debug_log; -use crate::optional_env; -use crate::target_os::TargetOs; - -pub fn configure_and_build(context: &BuildContext) -> Result { - let mut config = Config::new(&context.llama_src); - - configure_base_defines(&mut config); - configure_cpu_features( - &mut config, - &context.cargo_cfg_target_arch, - context.target_os, - )?; - configure_shared_libs(&mut config, context.build_shared_libs); - configure_platform_specific( - &mut config, - context.target_os, - &context.target_triple, - context.android_ndk.as_ref(), - ); - configure_gpu_backends(&mut config, context.target_os)?; - configure_openmp(&mut config, context.target_os); - configure_system_ggml(&mut config)?; - let backends_dir = configure_dynamic_backends(&mut config, &context.cmake_dir)?; - - config - .out_dir(&context.cmake_dir) - .profile(&context.profile) - .always_configure(false); - - let install_dir = config.build(); - - if let Some(dir) = backends_dir { - println!("cargo:backends_dir={}", dir.display()); - } - - Ok(install_dir) -} - -fn configure_dynamic_backends( - config: &mut Config, - cmake_dir: &Path, -) -> Result, BuildError> { - if !cfg!(feature = "dynamic-backends") { - return Ok(None); - } - - let backends_dir = cmake_dir.join("backends"); - - std::fs::create_dir_all(&backends_dir).map_err(|source| BuildError::Filesystem { - path: backends_dir.clone(), - source, - })?; - - config.define("GGML_BACKEND_DL", "ON"); - config.define("GGML_CPU_ALL_VARIANTS", "ON"); - config.define("GGML_BACKEND_DIR", &backends_dir); - - Ok(Some(backends_dir)) -} - -fn configure_base_defines(config: &mut Config) { - config.define("LLAMA_BUILD_TESTS", "OFF"); - config.define("LLAMA_BUILD_EXAMPLES", "OFF"); - config.define("LLAMA_BUILD_SERVER", "OFF"); - config.define("LLAMA_BUILD_TOOLS", "OFF"); - config.define("LLAMA_BUILD_APP", "OFF"); - config.define("LLAMA_BUILD_COMMON", "ON"); - config.define("LLAMA_BUILD_MTMD", "ON"); - config.define("MTMD_VIDEO", "OFF"); - config.define("LLAMA_CURL", "OFF"); -} - -fn configure_cpu_features( - config: &mut Config, - cargo_cfg_target_arch: &str, - target_os: TargetOs, -) -> Result<(), BuildError> { - let target_cpu = optional_env("CARGO_ENCODED_RUSTFLAGS")?.and_then(|rustflags| { - rustflags - .split('\x1f') - .find(|flag| flag.contains("target-cpu=")) - .and_then(|flag| flag.split("target-cpu=").nth(1)) - .map(ToString::to_string) - }); - - if target_cpu.as_deref() == Some("native") { - debug_log!("Detected target-cpu=native, compiling with GGML_NATIVE"); - config.define("GGML_NATIVE", "ON"); - - return Ok(()); - } - - config.define("GGML_NATIVE", "OFF"); - - if let Some(ref cpu) = target_cpu { - debug_log!("Setting baseline architecture: -march={}", cpu); - config.cflag(format!("-march={cpu}")); - config.cxxflag(format!("-march={cpu}")); - } - - let features = optional_env("CARGO_CFG_TARGET_FEATURE")?.unwrap_or_default(); - debug_log!("Compiling with target features: {}", features); - - for feature in features.split(',') { - if let Some(ggml_flag) = map_cpu_feature_to_ggml(feature) { - config.define(ggml_flag, "ON"); - } - } - - if cargo_cfg_target_arch == "aarch64" - && target_os == TargetOs::Linux - && target_cpu.as_deref() != Some("native") - { - config.define("GGML_CPU_ARM_ARCH", "armv8-a"); - } - - Ok(()) -} - -fn map_cpu_feature_to_ggml(feature: &str) -> Option<&'static str> { - match feature { - "avx" => Some("GGML_AVX"), - "avx2" => Some("GGML_AVX2"), - "avx512bf16" => Some("GGML_AVX512_BF16"), - "avx512vbmi" => Some("GGML_AVX512_VBMI"), - "avx512vnni" => Some("GGML_AVX512_VNNI"), - "avxvnni" => Some("GGML_AVX_VNNI"), - "bmi2" => Some("GGML_BMI2"), - "f16c" => Some("GGML_F16C"), - "fma" => Some("GGML_FMA"), - "sse4.2" => Some("GGML_SSE42"), - _ => None, - } -} - -fn configure_shared_libs(config: &mut Config, build_shared_libs: bool) { - config.define( - "BUILD_SHARED_LIBS", - if build_shared_libs { "ON" } else { "OFF" }, - ); -} - -fn configure_platform_specific( - config: &mut Config, - target_os: TargetOs, - target_triple: &str, - android_ndk: Option<&AndroidNdk>, -) { - match target_os { - TargetOs::Apple(_) => { - config.define("GGML_BLAS", "OFF"); - } - TargetOs::Android => { - if let Some(ndk) = android_ndk { - configure_android_cmake(config, ndk, target_triple); - } - } - TargetOs::Linux => {} - } -} - -fn configure_android_cmake(config: &mut Config, ndk: &AndroidNdk, _target_triple: &str) { - #[cfg(all(feature = "shared-stdcxx", feature = "static-stdcxx"))] - compile_error!("Features 'shared-stdcxx' and 'static-stdcxx' are mutually exclusive"); - - println!("cargo:rerun-if-env-changed=ANDROID_NDK_HOME"); - println!("cargo:rerun-if-env-changed=ANDROID_PLATFORM"); - - config.define("CMAKE_TOOLCHAIN_FILE", ndk.cmake_toolchain_file()); - config.define("ANDROID_PLATFORM", ndk.android_platform()); - config.define("ANDROID_ABI", ndk.abi); - - if cfg!(feature = "static-stdcxx") { - config.define("ANDROID_STL", "c++_static"); - } else if cfg!(feature = "shared-stdcxx") { - config.define("ANDROID_STL", "c++_shared"); - } - - config.define("GGML_LLAMAFILE", "OFF"); - - println!("cargo:rustc-link-lib=log"); - println!("cargo:rustc-link-lib=android"); -} - -fn configure_gpu_backends(config: &mut Config, target_os: TargetOs) -> Result<(), BuildError> { - if cfg!(feature = "vulkan") { - config.define("GGML_VULKAN", "ON"); - configure_vulkan_linking(target_os)?; - } - - if cfg!(feature = "cuda") { - config.define("GGML_CUDA", "ON"); - - if cfg!(feature = "cuda-no-vmm") { - config.define("GGML_CUDA_NO_VMM", "ON"); - } - - if let Some(cuda_architectures) = optional_env("CUDAARCHS")? { - config.define("CMAKE_CUDA_ARCHITECTURES", cuda_architectures); - } - } - - if cfg!(feature = "rocm") { - config.define("GGML_HIP", "ON"); - } - - Ok(()) -} - -fn configure_vulkan_linking(target_os: TargetOs) -> Result<(), BuildError> { - match target_os { - TargetOs::Linux => { - match env::var("VULKAN_SDK") { - Ok(vulkan_path) => { - let vulkan_lib_path = Path::new(&vulkan_path).join("lib"); - - println!("cargo:rustc-link-search={}", vulkan_lib_path.display()); - } - Err(env::VarError::NotPresent) => {} - Err(source) => { - return Err(BuildError::Environment { - name: "VULKAN_SDK", - source, - }); - } - } - - println!("cargo:rustc-link-lib=vulkan"); - } - TargetOs::Apple(_) | TargetOs::Android => {} - } - - Ok(()) -} - -fn configure_openmp(config: &mut Config, target_os: TargetOs) { - let openmp_enabled = cfg!(feature = "openmp") && !target_os.is_android(); - - config.define("GGML_OPENMP", if openmp_enabled { "ON" } else { "OFF" }); -} - -fn configure_system_ggml(config: &mut Config) -> Result<(), BuildError> { - if cfg!(feature = "system-ggml") { - println!("cargo:rerun-if-env-changed=GGML_DIR"); - let ggml_dir = env::var("GGML_DIR").map_err(|source| BuildError::Environment { - name: "GGML_DIR", - source, - })?; - config.define("LLAMA_USE_SYSTEM_GGML", "ON"); - config.define("GGML_DIR", ggml_dir); - } - - Ok(()) -} - -#[cfg(test)] -mod cpu_feature_mapping_tests { - use super::map_cpu_feature_to_ggml; - - #[test] - fn every_supported_rust_cpu_feature_maps_to_its_ggml_switch() { - assert_eq!(map_cpu_feature_to_ggml("avx"), Some("GGML_AVX")); - assert_eq!(map_cpu_feature_to_ggml("avx2"), Some("GGML_AVX2")); - assert_eq!( - map_cpu_feature_to_ggml("avx512bf16"), - Some("GGML_AVX512_BF16") - ); - assert_eq!( - map_cpu_feature_to_ggml("avx512vbmi"), - Some("GGML_AVX512_VBMI") - ); - assert_eq!( - map_cpu_feature_to_ggml("avx512vnni"), - Some("GGML_AVX512_VNNI") - ); - assert_eq!(map_cpu_feature_to_ggml("avxvnni"), Some("GGML_AVX_VNNI")); - assert_eq!(map_cpu_feature_to_ggml("bmi2"), Some("GGML_BMI2")); - assert_eq!(map_cpu_feature_to_ggml("f16c"), Some("GGML_F16C")); - assert_eq!(map_cpu_feature_to_ggml("fma"), Some("GGML_FMA")); - assert_eq!(map_cpu_feature_to_ggml("sse4.2"), Some("GGML_SSE42")); - } - - #[test] - fn unsupported_rust_cpu_feature_does_not_enable_a_ggml_switch() { - assert_eq!(map_cpu_feature_to_ggml("aes"), None); - assert_eq!(map_cpu_feature_to_ggml(""), None); - } -} diff --git a/llama-cpp-bindings-build/src/cmake_configuration.rs b/llama-cpp-bindings-build/src/cmake_configuration.rs new file mode 100644 index 000000000..248809f8a --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_configuration.rs @@ -0,0 +1,532 @@ +use std::path::PathBuf; + +use crate::android_ndk::AndroidNdk; +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cargo_directive::CargoDirective; +use crate::cmake_define::CmakeDefine; +use crate::cpu_tuning::CpuTuning; +use crate::link_kind::LinkKind; +use crate::native_library::NativeLibrary; +use crate::target_os::TargetOs; + +fn base_defines() -> Vec { + vec![ + CmakeDefine::new("LLAMA_BUILD_TESTS", "OFF"), + CmakeDefine::new("LLAMA_BUILD_EXAMPLES", "OFF"), + CmakeDefine::new("LLAMA_BUILD_SERVER", "OFF"), + CmakeDefine::new("LLAMA_BUILD_TOOLS", "OFF"), + CmakeDefine::new("LLAMA_BUILD_APP", "OFF"), + CmakeDefine::new("LLAMA_BUILD_COMMON", "ON"), + CmakeDefine::new("LLAMA_BUILD_MTMD", "ON"), + CmakeDefine::new("MTMD_VIDEO", "OFF"), + CmakeDefine::new("LLAMA_OPENSSL", "OFF"), + ] +} + +/// How llama.cpp is configured with `CMake` for this build, and the cargo directives that +/// configuration implies. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CmakeConfiguration { + pub defines: Vec, + pub required_directories: Vec, + pub directives: Vec, +} + +impl CmakeConfiguration { + /// # Errors + /// + /// Returns the [`BuildError`] of [`CpuTuning::of`], or [`BuildError::Environment`] when a + /// variable an enabled feature reads is not Unicode, or when `GGML_DIR` is missing for + /// `system-ggml`. + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Result { + let CpuTuning { + defines: cpu_defines, + debug_messages, + } = CpuTuning::of(context, environment)?; + let mut configuration = Self { + defines: [base_defines(), cpu_defines].concat(), + required_directories: Vec::new(), + directives: debug_messages, + }; + + configuration.defines.push(CmakeDefine::switch( + "BUILD_SHARED_LIBS", + context.build_shared_libs, + )); + configuration.configure_platform(context, environment); + configuration.configure_gpu_backends(context, environment)?; + configuration.defines.push(CmakeDefine::switch( + "GGML_OPENMP", + environment.has(BuildFeature::Openmp) + && !matches!(context.target_os, TargetOs::Android(_)), + )); + configuration.configure_system_ggml(environment)?; + configuration.configure_dynamic_backends(context, environment); + + Ok(configuration) + } + + fn configure_platform(&mut self, context: &BuildContext, environment: &BuildEnvironment) { + match &context.target_os { + TargetOs::Apple => self.defines.push(CmakeDefine::new("GGML_BLAS", "OFF")), + TargetOs::Android(android_ndk) => self.configure_android(android_ndk, environment), + TargetOs::Linux => {} + } + } + + fn configure_android(&mut self, android_ndk: &AndroidNdk, environment: &BuildEnvironment) { + self.directives.extend([ + CargoDirective::RerunIfEnvChanged("ANDROID_NDK_HOME".to_owned()), + CargoDirective::RerunIfEnvChanged("ANDROID_PLATFORM".to_owned()), + ]); + self.defines.extend([ + CmakeDefine::new("CMAKE_TOOLCHAIN_FILE", android_ndk.cmake_toolchain_file()), + CmakeDefine::new("ANDROID_PLATFORM", android_ndk.android_platform()), + CmakeDefine::new("ANDROID_ABI", android_ndk.abi), + ]); + + if environment.has(BuildFeature::StaticStdcxx) { + self.defines + .push(CmakeDefine::new("ANDROID_STL", "c++_static")); + } else if environment.has(BuildFeature::SharedStdcxx) { + self.defines + .push(CmakeDefine::new("ANDROID_STL", "c++_shared")); + } + + self.defines.push(CmakeDefine::new("GGML_LLAMAFILE", "OFF")); + self.directives.extend(["log", "android"].map(|name| { + CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Unspecified, + name, + }) + })); + } + + fn configure_gpu_backends( + &mut self, + context: &BuildContext, + environment: &BuildEnvironment, + ) -> Result<(), BuildError> { + if environment.has(BuildFeature::Vulkan) { + self.defines.push(CmakeDefine::new("GGML_VULKAN", "ON")); + + if context.target_os == TargetOs::Linux { + self.link_vulkan(environment)?; + } + } + + if environment.has(BuildFeature::Cuda) { + self.defines.push(CmakeDefine::new("GGML_CUDA", "ON")); + + if environment.has(BuildFeature::CudaNoVmm) { + self.defines + .push(CmakeDefine::new("GGML_CUDA_NO_VMM", "ON")); + } + + if let Some(cuda_architectures) = environment.optional("CUDAARCHS")? { + self.defines.push(CmakeDefine::new( + "CMAKE_CUDA_ARCHITECTURES", + cuda_architectures, + )); + } + } + + if environment.has(BuildFeature::Rocm) { + self.defines.push(CmakeDefine::new("GGML_HIP", "ON")); + } + + if context.target_os == TargetOs::Apple { + self.defines.push(CmakeDefine::switch( + "GGML_METAL", + environment.has(BuildFeature::Metal), + )); + } + + Ok(()) + } + + fn link_vulkan(&mut self, environment: &BuildEnvironment) -> Result<(), BuildError> { + self.directives + .push(CargoDirective::RerunIfEnvChanged("VULKAN_SDK".to_owned())); + + if let Some(vulkan_sdk) = environment.optional("VULKAN_SDK")? { + self.directives.push(CargoDirective::LinkSearch( + PathBuf::from(vulkan_sdk).join("lib"), + )); + } + + self.directives + .push(CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Unspecified, + name: "vulkan", + })); + + Ok(()) + } + + fn configure_system_ggml(&mut self, environment: &BuildEnvironment) -> Result<(), BuildError> { + if environment.has(BuildFeature::SystemGgml) { + self.directives + .push(CargoDirective::RerunIfEnvChanged("GGML_DIR".to_owned())); + + let ggml_dir = environment.required("GGML_DIR")?; + + self.defines.extend([ + CmakeDefine::new("LLAMA_USE_SYSTEM_GGML", "ON"), + CmakeDefine::new("GGML_DIR", ggml_dir), + ]); + } + + Ok(()) + } + + fn configure_dynamic_backends( + &mut self, + context: &BuildContext, + environment: &BuildEnvironment, + ) { + if environment.has(BuildFeature::DynamicBackends) { + let backends_dir = context.cmake_dir().join("backends"); + + self.defines.extend([ + CmakeDefine::new("GGML_BACKEND_DL", "ON"), + CmakeDefine::new("GGML_CPU_ALL_VARIANTS", "ON"), + CmakeDefine::new("GGML_BACKEND_DIR", backends_dir.clone()), + ]); + self.required_directories.push(backends_dir); + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use enumflags2::BitFlags; + + use super::CmakeConfiguration; + use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; + use crate::build_context::tests::android_overrides; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_environment::tests::EnvironmentVariable; + use crate::build_environment::tests::variable; + use crate::build_environment::tests::with_non_unicode; + use crate::build_feature::BuildFeature; + use crate::cargo_directive::CargoDirective; + use crate::cmake_define::CmakeDefine; + use crate::cpu_tuning::CpuTuning; + use crate::link_kind::LinkKind; + use crate::native_library::NativeLibrary; + + fn configured( + features: BitFlags, + overrides: &[EnvironmentVariable], + ) -> CmakeConfiguration { + let environment = cargo_environment(features, overrides); + + CmakeConfiguration::of(&context_of(&environment), &environment).unwrap() + } + + fn after_the_base_switches( + features: BitFlags, + overrides: &[EnvironmentVariable], + ) -> CmakeConfiguration { + let environment = cargo_environment(features, overrides); + let context = context_of(&environment); + let cpu_defines = CpuTuning::of(&context, &environment).unwrap().defines; + let mut configuration = CmakeConfiguration::of(&context, &environment).unwrap(); + + configuration.defines.drain(..9 + cpu_defines.len()); + + configuration + } + + fn compiling_with_no_target_features() -> CargoDirective { + CargoDirective::Debug("Compiling with target features: ".to_owned()) + } + + fn unspecified_link(name: &'static str) -> CargoDirective { + CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Unspecified, + name, + }) + } + + #[test] + fn a_plain_linux_build_turns_off_everything_but_the_libraries_the_bindings_use() { + assert_eq!( + configured(BitFlags::empty(), &[]), + CmakeConfiguration { + defines: vec![ + CmakeDefine::new("LLAMA_BUILD_TESTS", "OFF"), + CmakeDefine::new("LLAMA_BUILD_EXAMPLES", "OFF"), + CmakeDefine::new("LLAMA_BUILD_SERVER", "OFF"), + CmakeDefine::new("LLAMA_BUILD_TOOLS", "OFF"), + CmakeDefine::new("LLAMA_BUILD_APP", "OFF"), + CmakeDefine::new("LLAMA_BUILD_COMMON", "ON"), + CmakeDefine::new("LLAMA_BUILD_MTMD", "ON"), + CmakeDefine::new("MTMD_VIDEO", "OFF"), + CmakeDefine::new("LLAMA_OPENSSL", "OFF"), + CmakeDefine::new("GGML_NATIVE", "OFF"), + CmakeDefine::new("GGML_SSE42", "OFF"), + CmakeDefine::new("GGML_AVX", "OFF"), + CmakeDefine::new("GGML_AVX_VNNI", "OFF"), + CmakeDefine::new("GGML_AVX2", "OFF"), + CmakeDefine::new("GGML_BMI2", "OFF"), + CmakeDefine::new("GGML_FMA", "OFF"), + CmakeDefine::new("GGML_F16C", "OFF"), + CmakeDefine::new("GGML_AVX512", "OFF"), + CmakeDefine::new("GGML_AVX512_VBMI", "OFF"), + CmakeDefine::new("GGML_AVX512_VNNI", "OFF"), + CmakeDefine::new("GGML_AVX512_BF16", "OFF"), + CmakeDefine::new("GGML_AMX_TILE", "OFF"), + CmakeDefine::new("GGML_AMX_INT8", "OFF"), + CmakeDefine::new("GGML_AMX_BF16", "OFF"), + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + ], + required_directories: Vec::new(), + directives: vec![compiling_with_no_target_features()], + } + ); + } + + #[test] + fn apple_targets_build_without_blas_and_leave_vulkan_linking_to_the_platform() { + assert_eq!( + after_the_base_switches( + BuildFeature::Metal | BuildFeature::Vulkan, + &[variable("CARGO_CFG_TARGET_OS", "macos")] + ), + CmakeConfiguration { + defines: vec![ + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_BLAS", "OFF"), + CmakeDefine::new("GGML_VULKAN", "ON"), + CmakeDefine::new("GGML_METAL", "ON"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + ], + required_directories: Vec::new(), + directives: vec![compiling_with_no_target_features()], + } + ); + } + + #[test] + fn android_targets_build_with_the_ndk_toolchain_their_stl_and_no_openmp() { + let android_with = |features: BitFlags| { + after_the_base_switches(features | BuildFeature::Openmp, &android_overrides()) + }; + let android_defines = |stl: Vec| { + [ + vec![ + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new( + "CMAKE_TOOLCHAIN_FILE", + format!("{FIXTURE_ANDROID_NDK}/build/cmake/android.toolchain.cmake"), + ), + CmakeDefine::new("ANDROID_PLATFORM", "android-28"), + CmakeDefine::new("ANDROID_ABI", "arm64-v8a"), + ], + stl, + vec![ + CmakeDefine::new("GGML_LLAMAFILE", "OFF"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + ], + ] + .concat() + }; + let android_configuration = |stl: Vec| CmakeConfiguration { + defines: android_defines(stl), + required_directories: Vec::new(), + directives: vec![ + compiling_with_no_target_features(), + CargoDirective::RerunIfEnvChanged("ANDROID_NDK_HOME".to_owned()), + CargoDirective::RerunIfEnvChanged("ANDROID_PLATFORM".to_owned()), + unspecified_link("log"), + unspecified_link("android"), + ], + }; + + assert_eq!( + [ + android_with(BitFlags::from(BuildFeature::StaticStdcxx)), + android_with(BitFlags::from(BuildFeature::SharedStdcxx)), + android_with(BitFlags::empty()), + ], + [ + android_configuration(vec![CmakeDefine::new("ANDROID_STL", "c++_static")]), + android_configuration(vec![CmakeDefine::new("ANDROID_STL", "c++_shared")]), + android_configuration(Vec::new()), + ] + ); + } + + #[test] + fn gpu_backends_turn_on_their_ggml_switches_and_take_their_sdk_and_architectures_when_set() { + let gpu_configuration = |overrides: &[EnvironmentVariable]| { + after_the_base_switches( + BuildFeature::Vulkan + | BuildFeature::Cuda + | BuildFeature::CudaNoVmm + | BuildFeature::Rocm, + overrides, + ) + }; + let gpu_defines = |cuda_architectures: Vec| { + [ + vec![ + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_VULKAN", "ON"), + CmakeDefine::new("GGML_CUDA", "ON"), + CmakeDefine::new("GGML_CUDA_NO_VMM", "ON"), + ], + cuda_architectures, + vec![ + CmakeDefine::new("GGML_HIP", "ON"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + ], + ] + .concat() + }; + let vulkan_directives = |vulkan_sdk_search_path: Vec| { + [ + vec![ + compiling_with_no_target_features(), + CargoDirective::RerunIfEnvChanged("VULKAN_SDK".to_owned()), + ], + vulkan_sdk_search_path, + vec![unspecified_link("vulkan")], + ] + .concat() + }; + + assert_eq!( + [ + gpu_configuration(&[ + variable("VULKAN_SDK", "/vulkan"), + variable("CUDAARCHS", "89") + ]), + gpu_configuration(&[]), + ], + [ + CmakeConfiguration { + defines: gpu_defines(vec![CmakeDefine::new("CMAKE_CUDA_ARCHITECTURES", "89")]), + required_directories: Vec::new(), + directives: vulkan_directives(vec![CargoDirective::LinkSearch(PathBuf::from( + "/vulkan/lib" + ))]), + }, + CmakeConfiguration { + defines: gpu_defines(Vec::new()), + required_directories: Vec::new(), + directives: vulkan_directives(Vec::new()), + }, + ] + ); + } + + #[test] + fn openmp_is_turned_on_for_linux() { + assert_eq!( + after_the_base_switches(BitFlags::from(BuildFeature::Openmp), &[]).defines, + [ + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_OPENMP", "ON"), + ] + ); + } + + #[test] + fn system_ggml_builds_llama_cpp_against_the_ggml_in_ggml_dir() { + assert_eq!( + after_the_base_switches( + BitFlags::from(BuildFeature::SystemGgml), + &[variable("GGML_DIR", "/ggml/lib/cmake/ggml")] + ), + CmakeConfiguration { + defines: vec![ + CmakeDefine::new("BUILD_SHARED_LIBS", "OFF"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + CmakeDefine::new("LLAMA_USE_SYSTEM_GGML", "ON"), + CmakeDefine::new("GGML_DIR", "/ggml/lib/cmake/ggml"), + ], + required_directories: Vec::new(), + directives: vec![ + compiling_with_no_target_features(), + CargoDirective::RerunIfEnvChanged("GGML_DIR".to_owned()), + ], + } + ); + } + + #[test] + fn dynamic_backends_are_built_as_shared_libraries_installed_into_a_published_backends_dir() { + assert_eq!( + after_the_base_switches( + BuildFeature::DynamicBackends | BuildFeature::DynamicLink, + &[] + ), + CmakeConfiguration { + defines: vec![ + CmakeDefine::new("BUILD_SHARED_LIBS", "ON"), + CmakeDefine::new("GGML_OPENMP", "OFF"), + CmakeDefine::new("GGML_BACKEND_DL", "ON"), + CmakeDefine::new("GGML_CPU_ALL_VARIANTS", "ON"), + CmakeDefine::new("GGML_BACKEND_DIR", "/out/cmake/backends"), + ], + required_directories: vec![PathBuf::from("/out/cmake/backends")], + directives: vec![CargoDirective::Debug( + "Building every CPU variant for ggml to select at runtime".to_owned() + )], + } + ); + } + + #[test] + fn the_configuration_refuses_non_unicode_variables_and_a_missing_ggml_dir() { + let context = context_of(&cargo_environment(BitFlags::empty(), &[])); + let refusal = |features: BitFlags, name: &str| { + CmakeConfiguration::of( + &context, + &with_non_unicode(cargo_environment(features, &[]), name), + ) + .err() + .map(|error| error.to_string()) + }; + let not_unicode = |name: &str| { + Some(format!( + "environment variable {name} could not be read: environment variable was not \ + valid unicode: \"\\xFF\"" + )) + }; + + assert_eq!( + [ + refusal(BitFlags::empty(), "CARGO_ENCODED_RUSTFLAGS"), + refusal(BitFlags::from(BuildFeature::Vulkan), "VULKAN_SDK"), + refusal(BitFlags::from(BuildFeature::Cuda), "CUDAARCHS"), + CmakeConfiguration::of( + &context, + &cargo_environment(BitFlags::from(BuildFeature::SystemGgml), &[]) + ) + .err() + .map(|error| error.to_string()), + ], + [ + not_unicode("CARGO_ENCODED_RUSTFLAGS"), + not_unicode("VULKAN_SDK"), + not_unicode("CUDAARCHS"), + Some( + "environment variable GGML_DIR could not be read: environment variable not \ + found" + .to_owned() + ), + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/cmake_define.rs b/llama-cpp-bindings-build/src/cmake_define.rs new file mode 100644 index 000000000..47edb4656 --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_define.rs @@ -0,0 +1,22 @@ +use std::ffi::OsString; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CmakeDefine { + pub name: &'static str, + pub value: OsString, +} + +impl CmakeDefine { + #[must_use] + pub fn new(name: &'static str, value: impl Into) -> Self { + Self { + name, + value: value.into(), + } + } + + #[must_use] + pub fn switch(name: &'static str, enabled: bool) -> Self { + Self::new(name, if enabled { "ON" } else { "OFF" }) + } +} diff --git a/llama-cpp-bindings-build/src/cmake_file_api.rs b/llama-cpp-bindings-build/src/cmake_file_api.rs new file mode 100644 index 000000000..e0fa22ad7 --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_file_api.rs @@ -0,0 +1,135 @@ +use std::fs; +use std::fs::DirEntry; +use std::io; +use std::path::Path; +use std::path::PathBuf; + +use serde::de::DeserializeOwned; + +use crate::build_error::BuildError; +use crate::cmake_cache::CmakeCache; +use crate::cmake_reply_index::CmakeReplyIndex; + +const REPLY_INDEX_PREFIX: &[u8] = b"index-"; + +fn read_reply_file(path: PathBuf) -> Result { + let contents = + fs::read(&path).map_err(|source| BuildError::CmakeFileApiReplyFileUnreadable { + path: path.clone(), + source, + })?; + + serde_json::from_slice(&contents) + .map_err(|source| BuildError::CmakeFileApiReplyFileInvalid { path, source }) +} + +fn current_reply_index(reply_dir: &Path) -> Result { + fs::read_dir(reply_dir) + .and_then(Iterator::collect::, io::Error>>) + .map_err(|source| BuildError::CmakeFileApiReplyDirectoryUnreadable { + path: reply_dir.to_path_buf(), + source, + })? + .into_iter() + .map(|entry| entry.file_name()) + .filter(|file_name| file_name.as_encoded_bytes().starts_with(REPLY_INDEX_PREFIX)) + .max() + .map(|file_name| reply_dir.join(file_name)) + .ok_or_else(|| BuildError::CmakeFileApiReplyIndexMissing { + path: reply_dir.to_path_buf(), + }) +} + +/// The `CMake` file API of the build tree in `build_dir`, through which the build learns what +/// `CMake` resolved while configuring. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CmakeFileApi { + pub build_dir: PathBuf, +} + +impl CmakeFileApi { + /// The initial-cache script that, passed to `CMake` with `-C`, has every configure of the + /// build tree query its cache before `CMake` reads the queries. + pub const CACHE_QUERY_SCRIPT: &'static str = + concat!(env!("CARGO_MANIFEST_DIR"), "/cmake/cache_query.cmake"); + + /// The cache `CMake` reported in its current reply to [`Self::CACHE_QUERY_SCRIPT`]. + /// + /// # Errors + /// + /// Returns [`BuildError::CmakeFileApiReplyDirectoryUnreadable`] when `CMake` wrote no + /// reply, [`BuildError::CmakeFileApiReplyIndexMissing`] when the reply has no index, or + /// [`BuildError::CmakeFileApiReplyFileUnreadable`] and + /// [`BuildError::CmakeFileApiReplyFileInvalid`] when a reply file cannot be read as the + /// reply this crate queried. + pub fn cache(&self) -> Result { + let reply_dir = self.build_dir.join(".cmake/api/v1/reply"); + let CmakeReplyIndex { reply } = read_reply_file(current_reply_index(&reply_dir)?)?; + + read_reply_file(reply_dir.join(reply.cache.json_file)) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::CmakeFileApi; + + const FIXTURES: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/cmake_file_api"); + + fn cache_failure(build_tree: &str) -> Option { + CmakeFileApi { + build_dir: PathBuf::from(FIXTURES).join(build_tree), + } + .cache() + .err() + .map(|error| error.to_string()) + } + + #[test] + fn a_build_tree_cmake_never_replied_to_is_refused() { + assert_eq!( + cache_failure("never_replied"), + Some(format!( + "the CMake file API reply directory {FIXTURES}/never_replied/.cmake/api/v1/reply \ + could not be listed: No such file or directory (os error 2)" + )) + ); + } + + #[test] + fn a_reply_without_an_index_is_refused() { + assert_eq!( + cache_failure("reply_without_index"), + Some(format!( + "CMake wrote no file API reply index into \ + {FIXTURES}/reply_without_index/.cmake/api/v1/reply" + )) + ); + } + + #[test] + fn an_index_naming_a_missing_reply_file_is_refused() { + assert_eq!( + cache_failure("index_naming_a_missing_cache"), + Some(format!( + "the CMake file API reply file \ + {FIXTURES}/index_naming_a_missing_cache/.cmake/api/v1/reply/cache-v2-missing.json \ + could not be read: No such file or directory (os error 2)" + )) + ); + } + + #[test] + fn an_index_without_a_cache_reply_is_refused() { + assert_eq!( + cache_failure("index_without_cache_reply"), + Some(format!( + "the CMake file API reply file \ + {FIXTURES}/index_without_cache_reply/.cmake/api/v1/reply/index-0.json is not the \ + reply this crate queried: missing field `cache-v2` at line 1 column 11" + )) + ); + } +} diff --git a/llama-cpp-bindings-build/src/cmake_replies.rs b/llama-cpp-bindings-build/src/cmake_replies.rs new file mode 100644 index 000000000..8e621a740 --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_replies.rs @@ -0,0 +1,10 @@ +use serde::Deserialize; + +use crate::cmake_reply_file::CmakeReplyFile; + +/// The replies `CMake` wrote for the shared stateless queries this crate submits. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeReplies { + #[serde(rename = "cache-v2")] + pub cache: CmakeReplyFile, +} diff --git a/llama-cpp-bindings-build/src/cmake_reply_file.rs b/llama-cpp-bindings-build/src/cmake_reply_file.rs new file mode 100644 index 000000000..86a0c7b7a --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_reply_file.rs @@ -0,0 +1,10 @@ +use std::path::PathBuf; + +use serde::Deserialize; + +/// A reference from the file API's reply index to the reply file holding one object. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeReplyFile { + #[serde(rename = "jsonFile")] + pub json_file: PathBuf, +} diff --git a/llama-cpp-bindings-build/src/cmake_reply_index.rs b/llama-cpp-bindings-build/src/cmake_reply_index.rs new file mode 100644 index 000000000..3d1bc3766 --- /dev/null +++ b/llama-cpp-bindings-build/src/cmake_reply_index.rs @@ -0,0 +1,9 @@ +use serde::Deserialize; + +use crate::cmake_replies::CmakeReplies; + +/// The file API's reply index, which names the reply file answering each query. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub struct CmakeReplyIndex { + pub reply: CmakeReplies, +} diff --git a/llama-cpp-bindings-build/src/cpp_wrapper.rs b/llama-cpp-bindings-build/src/cpp_wrapper.rs deleted file mode 100644 index 2c1bc17c7..000000000 --- a/llama-cpp-bindings-build/src/cpp_wrapper.rs +++ /dev/null @@ -1,30 +0,0 @@ -use llama_cpp_wrapper_sources::cpp_standard::CPP_STANDARD; -use llama_cpp_wrapper_sources::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; -use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; - -use crate::BuildError; -use crate::target_os::TargetOs; - -pub fn compile_cpp_wrappers(target_os: TargetOs) -> Result<(), BuildError> { - let mut build = cc::Build::new(); - - build.cpp(true).pic(true); - - for include_dir in WRAPPER_INCLUDE_DIRS { - build.include(include_dir); - } - - build.flag_if_supported(format!("-std={CPP_STANDARD}")); - - for source in WRAPPER_SOURCES { - build.file(source); - } - - if target_os.is_android() && cfg!(feature = "static-stdcxx") { - build.cpp_link_stdlib(None); - } - - build - .try_compile("llama_cpp_bindings_sys_common_wrapper") - .map_err(BuildError::NativeWrapper) -} diff --git a/llama-cpp-bindings-build/src/cpu_tuning.rs b/llama-cpp-bindings-build/src/cpu_tuning.rs new file mode 100644 index 000000000..b0ab197d5 --- /dev/null +++ b/llama-cpp-bindings-build/src/cpu_tuning.rs @@ -0,0 +1,373 @@ +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cargo_directive::CargoDirective; +use crate::cmake_define::CmakeDefine; +use crate::x86_ggml_switch::X86GgmlSwitch; + +fn codegen_options(encoded_rustflags: &str) -> Vec<&str> { + let mut options = Vec::new(); + let mut flags = encoded_rustflags.split('\x1f'); + + while let Some(flag) = flags.next() { + if flag == "-C" || flag == "--codegen" { + options.extend(flags.next()); + } else if let Some(option) = flag + .strip_prefix("--codegen=") + .or_else(|| flag.strip_prefix("-C")) + { + options.push(option); + } + } + + options +} + +fn target_cpu_of(encoded_rustflags: &str) -> Option { + codegen_options(encoded_rustflags) + .into_iter() + .rev() + .find_map(|option| option.strip_prefix("target-cpu=")) + .map(ToOwned::to_owned) +} + +const X86_GGML_SWITCHES: [X86GgmlSwitch; 14] = [ + X86GgmlSwitch { + name: "GGML_SSE42", + target_features: &["sse4.2"], + }, + X86GgmlSwitch { + name: "GGML_AVX", + target_features: &["avx"], + }, + X86GgmlSwitch { + name: "GGML_AVX_VNNI", + target_features: &["avxvnni"], + }, + X86GgmlSwitch { + name: "GGML_AVX2", + target_features: &["avx2"], + }, + X86GgmlSwitch { + name: "GGML_BMI2", + target_features: &["bmi2"], + }, + X86GgmlSwitch { + name: "GGML_FMA", + target_features: &["fma"], + }, + X86GgmlSwitch { + name: "GGML_F16C", + target_features: &["f16c"], + }, + X86GgmlSwitch { + name: "GGML_AVX512", + target_features: &["avx512f", "avx512cd", "avx512vl", "avx512dq", "avx512bw"], + }, + X86GgmlSwitch { + name: "GGML_AVX512_VBMI", + target_features: &["avx512vbmi"], + }, + X86GgmlSwitch { + name: "GGML_AVX512_VNNI", + target_features: &["avx512vnni"], + }, + X86GgmlSwitch { + name: "GGML_AVX512_BF16", + target_features: &["avx512bf16"], + }, + X86GgmlSwitch { + name: "GGML_AMX_TILE", + target_features: &["amx-tile"], + }, + X86GgmlSwitch { + name: "GGML_AMX_INT8", + target_features: &["amx-int8"], + }, + X86GgmlSwitch { + name: "GGML_AMX_BF16", + target_features: &["amx-bf16"], + }, +]; + +/// How ggml is compiled for the CPU rustc compiles for, and the debug messages explaining it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CpuTuning { + pub defines: Vec, + pub debug_messages: Vec, +} + +impl CpuTuning { + /// ggml builds every CPU variant and selects one at runtime when backends load dynamically. + /// Otherwise `target-cpu=native` lets ggml detect the CPU, and any other build switches + /// every x86 instruction set explicitly on or off, the way rustc's target features describe + /// the CPU, because ggml would enable AVX2-class defaults on its own. + /// + /// # Errors + /// + /// Returns [`BuildError::Environment`] when the rustflags or the target features are not + /// Unicode. + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Result { + if environment.has(BuildFeature::DynamicBackends) { + return Ok(Self { + defines: vec![CmakeDefine::new("GGML_NATIVE", "OFF")], + debug_messages: vec![CargoDirective::Debug( + "Building every CPU variant for ggml to select at runtime".to_owned(), + )], + }); + } + + let target_cpu = environment + .optional("CARGO_ENCODED_RUSTFLAGS")? + .and_then(|rustflags| target_cpu_of(&rustflags)); + + if target_cpu.as_deref() == Some("native") { + return Ok(Self { + defines: vec![CmakeDefine::new("GGML_NATIVE", "ON")], + debug_messages: vec![CargoDirective::Debug( + "Detected target-cpu=native, compiling with GGML_NATIVE".to_owned(), + )], + }); + } + + let target_features = environment + .optional("CARGO_CFG_TARGET_FEATURE")? + .unwrap_or_default(); + let enabled_features = target_features.split(',').collect::>(); + let mut defines = vec![CmakeDefine::new("GGML_NATIVE", "OFF")]; + + if matches!(context.cargo_cfg_target_arch.as_str(), "x86" | "x86_64") { + defines.extend( + X86_GGML_SWITCHES + .iter() + .map(|switch| switch.define_for(&enabled_features)), + ); + } + + Ok(Self { + defines, + debug_messages: vec![CargoDirective::Debug(format!( + "Compiling with target features: {target_features}" + ))], + }) + } +} + +#[cfg(test)] +mod target_cpu_of_tests { + use super::target_cpu_of; + + #[test] + fn an_attached_codegen_target_cpu_is_found() { + assert_eq!( + [ + target_cpu_of("-Ctarget-cpu=native"), + target_cpu_of("--codegen=target-cpu=znver4"), + ], + [Some("native".to_owned()), Some("znver4".to_owned())] + ); + } + + #[test] + fn a_codegen_target_cpu_given_as_the_next_flag_is_found() { + assert_eq!( + [ + target_cpu_of("-C\x1ftarget-cpu=znver4"), + target_cpu_of("--codegen\x1ftarget-cpu=znver4"), + ], + [Some("znver4".to_owned()), Some("znver4".to_owned())] + ); + } + + #[test] + fn the_last_target_cpu_wins_as_it_does_for_rustc() { + assert_eq!( + target_cpu_of("-Ctarget-cpu=haswell\x1f-Ctarget-cpu=znver4"), + Some("znver4".to_owned()) + ); + } + + #[test] + fn a_target_cpu_inside_another_codegen_option_is_not_the_target_cpu() { + assert_eq!(target_cpu_of("-Clink-arg=-Wl,target-cpu=znver4"), None); + } + + #[test] + fn rustflags_without_a_target_cpu_name_no_target_cpu() { + assert_eq!( + target_cpu_of("-Copt-level=3\x1f--cfg\x1ffeature=\"x\""), + None + ); + } +} + +#[cfg(test)] +mod cpu_tuning_tests { + use enumflags2::BitFlags; + + use super::CpuTuning; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_environment::tests::EnvironmentVariable; + use crate::build_environment::tests::variable; + use crate::build_environment::tests::with_non_unicode; + use crate::build_feature::BuildFeature; + use crate::cargo_directive::CargoDirective; + use crate::cmake_define::CmakeDefine; + + fn tuned(features: BitFlags, overrides: &[EnvironmentVariable]) -> CpuTuning { + let environment = cargo_environment(features, overrides); + + CpuTuning::of(&context_of(&environment), &environment).unwrap() + } + + fn x86_switches(enabled: &[&'static str]) -> Vec { + [ + "GGML_SSE42", + "GGML_AVX", + "GGML_AVX_VNNI", + "GGML_AVX2", + "GGML_BMI2", + "GGML_FMA", + "GGML_F16C", + "GGML_AVX512", + "GGML_AVX512_VBMI", + "GGML_AVX512_VNNI", + "GGML_AVX512_BF16", + "GGML_AMX_TILE", + "GGML_AMX_INT8", + "GGML_AMX_BF16", + ] + .into_iter() + .map(|name| CmakeDefine::switch(name, enabled.contains(&name))) + .collect() + } + + #[test] + fn a_native_target_cpu_lets_ggml_detect_the_cpu() { + assert_eq!( + tuned( + BitFlags::empty(), + &[ + variable("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), + variable("CARGO_CFG_TARGET_FEATURE", "avx2"), + ] + ), + CpuTuning { + defines: vec![CmakeDefine::new("GGML_NATIVE", "ON")], + debug_messages: vec![CargoDirective::Debug( + "Detected target-cpu=native, compiling with GGML_NATIVE".to_owned() + )], + } + ); + } + + #[test] + fn a_baseline_x86_build_turns_every_instruction_set_ggml_would_assume_off() { + assert_eq!( + tuned( + BitFlags::empty(), + &[variable("CARGO_CFG_TARGET_FEATURE", "fxsr,sse,sse2")] + ), + CpuTuning { + defines: [ + vec![CmakeDefine::new("GGML_NATIVE", "OFF")], + x86_switches(&[]), + ] + .concat(), + debug_messages: vec![CargoDirective::Debug( + "Compiling with target features: fxsr,sse,sse2".to_owned() + )], + } + ); + } + + #[test] + fn a_named_target_cpu_switches_on_exactly_the_instruction_sets_its_target_features_enable() { + assert_eq!( + tuned( + BitFlags::empty(), + &[ + variable("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=znver4"), + variable( + "CARGO_CFG_TARGET_FEATURE", + "avx,avx2,avx512f,avx512cd,avx512vl,avx512dq,bmi2,f16c,fma,sse4.2" + ), + ] + ) + .defines, + [ + vec![CmakeDefine::new("GGML_NATIVE", "OFF")], + x86_switches(&[ + "GGML_SSE42", + "GGML_AVX", + "GGML_AVX2", + "GGML_BMI2", + "GGML_FMA", + "GGML_F16C", + ]), + ] + .concat() + ); + } + + #[test] + fn an_aarch64_build_leaves_the_architecture_to_the_compiler_baseline() { + assert_eq!( + tuned( + BitFlags::empty(), + &[ + variable("CARGO_CFG_TARGET_ARCH", "aarch64"), + variable("TARGET", "aarch64-unknown-linux-gnu"), + ] + ), + CpuTuning { + defines: vec![CmakeDefine::new("GGML_NATIVE", "OFF")], + debug_messages: vec![CargoDirective::Debug( + "Compiling with target features: ".to_owned() + )], + } + ); + } + + #[test] + fn dynamic_backends_build_every_cpu_variant_without_tuning() { + assert_eq!( + tuned( + BuildFeature::DynamicBackends | BuildFeature::DynamicLink, + &[ + variable("CARGO_CFG_TARGET_ARCH", "aarch64"), + variable("TARGET", "aarch64-unknown-linux-gnu"), + variable("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"), + ] + ), + CpuTuning { + defines: vec![CmakeDefine::new("GGML_NATIVE", "OFF")], + debug_messages: vec![CargoDirective::Debug( + "Building every CPU variant for ggml to select at runtime".to_owned() + )], + } + ); + } + + #[test] + fn rustflags_or_target_features_that_are_not_unicode_are_refused() { + let context = context_of(&cargo_environment(BitFlags::empty(), &[])); + + assert_eq!( + ["CARGO_ENCODED_RUSTFLAGS", "CARGO_CFG_TARGET_FEATURE"].map(|name| { + CpuTuning::of( + &context, + &with_non_unicode(cargo_environment(BitFlags::empty(), &[]), name), + ) + .err() + .map(|error| error.to_string()) + }), + ["CARGO_ENCODED_RUSTFLAGS", "CARGO_CFG_TARGET_FEATURE"].map(|name| Some(format!( + "environment variable {name} could not be read: environment variable was not \ + valid unicode: \"\\xFF\"" + ))) + ); + } +} diff --git a/llama-cpp-bindings-build/src/emit_directives.rs b/llama-cpp-bindings-build/src/emit_directives.rs new file mode 100644 index 000000000..cc06f2b41 --- /dev/null +++ b/llama-cpp-bindings-build/src/emit_directives.rs @@ -0,0 +1,73 @@ +use std::io::Write; + +use crate::build_error::BuildError; +use crate::cargo_directive::CargoDirective; + +/// Writes `directives` for cargo, one per line, leaving the debug messages out unless +/// `include_debug` asks for them. +/// +/// # Errors +/// +/// Returns [`BuildError::DirectiveWrite`] when `output` refuses a line. +pub fn emit_directives( + directives: &[CargoDirective], + include_debug: bool, + output: &mut impl Write, +) -> Result<(), BuildError> { + directives + .iter() + .filter(|directive| include_debug || !directive.is_debug()) + .try_for_each(|directive| writeln!(output, "{directive}")) + .map_err(BuildError::DirectiveWrite) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::path::PathBuf; + + use super::emit_directives; + use crate::cargo_directive::CargoDirective; + + fn rebuild_and_debug_directives() -> [CargoDirective; 2] { + [ + CargoDirective::RerunIfChanged(PathBuf::from("wrapper.h")), + CargoDirective::Debug("Bindings Created".to_owned()), + ] + } + + fn emitted(include_debug: bool) -> String { + let mut output = Vec::new(); + + emit_directives(&rebuild_and_debug_directives(), include_debug, &mut output).unwrap(); + + String::from_utf8(output).unwrap() + } + + #[test] + fn debug_messages_are_left_out_unless_asked_for() { + assert_eq!( + [emitted(false), emitted(true)], + [ + "cargo:rerun-if-changed=wrapper.h\n".to_owned(), + "cargo:rerun-if-changed=wrapper.h\ncargo:warning=[DEBUG] Bindings Created\n" + .to_owned() + ] + ); + } + + #[test] + fn an_output_that_refuses_writes_is_reported() { + let mut read_only = File::open(concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml")).unwrap(); + + assert_eq!( + emit_directives(&rebuild_and_debug_directives(), false, &mut read_only) + .err() + .map(|error| error.to_string()), + Some( + "cargo directives could not be written: Bad file descriptor (os error 9)" + .to_owned() + ) + ); + } +} diff --git a/llama-cpp-bindings-build/src/enabled_build_features.rs b/llama-cpp-bindings-build/src/enabled_build_features.rs new file mode 100644 index 000000000..37a1d89ab --- /dev/null +++ b/llama-cpp-bindings-build/src/enabled_build_features.rs @@ -0,0 +1,81 @@ +use enumflags2::BitFlags; + +use crate::build_feature::BuildFeature; + +#[cfg(all(feature = "shared-stdcxx", feature = "static-stdcxx"))] +compile_error!("Features 'shared-stdcxx' and 'static-stdcxx' are mutually exclusive"); + +/// The cargo features this build crate was compiled with. +#[must_use] +pub fn enabled_build_features() -> BitFlags { + let mut features = BitFlags::empty(); + + features.set(BuildFeature::Cuda, cfg!(feature = "cuda")); + features.set(BuildFeature::CudaNoVmm, cfg!(feature = "cuda-no-vmm")); + features.set( + BuildFeature::DynamicBackends, + cfg!(feature = "dynamic-backends"), + ); + features.set(BuildFeature::DynamicLink, cfg!(feature = "dynamic-link")); + features.set( + BuildFeature::FaultInjection, + cfg!(feature = "fault-injection"), + ); + features.set(BuildFeature::Metal, cfg!(feature = "metal")); + features.set(BuildFeature::Openmp, cfg!(feature = "openmp")); + features.set(BuildFeature::Rocm, cfg!(feature = "rocm")); + features.set(BuildFeature::SharedStdcxx, cfg!(feature = "shared-stdcxx")); + features.set(BuildFeature::StaticStdcxx, cfg!(feature = "static-stdcxx")); + features.set(BuildFeature::SystemGgml, cfg!(feature = "system-ggml")); + features.set( + BuildFeature::SystemGgmlStatic, + cfg!(feature = "system-ggml-static"), + ); + features.set(BuildFeature::Vulkan, cfg!(feature = "vulkan")); + + features +} + +#[cfg(test)] +mod tests { + use super::enabled_build_features; + use crate::build_feature::BuildFeature; + + #[test] + fn every_build_feature_mirrors_its_cargo_feature() { + let features = enabled_build_features(); + + assert_eq!( + [ + features.contains(BuildFeature::Cuda), + features.contains(BuildFeature::CudaNoVmm), + features.contains(BuildFeature::DynamicBackends), + features.contains(BuildFeature::DynamicLink), + features.contains(BuildFeature::FaultInjection), + features.contains(BuildFeature::Metal), + features.contains(BuildFeature::Openmp), + features.contains(BuildFeature::Rocm), + features.contains(BuildFeature::SharedStdcxx), + features.contains(BuildFeature::StaticStdcxx), + features.contains(BuildFeature::SystemGgml), + features.contains(BuildFeature::SystemGgmlStatic), + features.contains(BuildFeature::Vulkan), + ], + [ + cfg!(feature = "cuda"), + cfg!(feature = "cuda-no-vmm"), + cfg!(feature = "dynamic-backends"), + cfg!(feature = "dynamic-link"), + cfg!(feature = "fault-injection"), + cfg!(feature = "metal"), + cfg!(feature = "openmp"), + cfg!(feature = "rocm"), + cfg!(feature = "shared-stdcxx"), + cfg!(feature = "static-stdcxx"), + cfg!(feature = "system-ggml"), + cfg!(feature = "system-ggml-static"), + cfg!(feature = "vulkan"), + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/host_os.rs b/llama-cpp-bindings-build/src/host_os.rs new file mode 100644 index 000000000..d5e4ef85a --- /dev/null +++ b/llama-cpp-bindings-build/src/host_os.rs @@ -0,0 +1,72 @@ +use std::env::consts::OS; + +use crate::build_error::BuildError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HostOs { + Linux, + MacOs, + Unsupported, +} + +impl HostOs { + /// The operating system the build script runs on, named by [`std::env::consts::OS`]. + #[must_use] + pub fn current() -> Self { + Self::named(OS) + } + + fn named(operating_system: &str) -> Self { + match operating_system { + "linux" => Self::Linux, + "macos" => Self::MacOs, + _ => Self::Unsupported, + } + } + + /// The directory name of the prebuilt Android NDK toolchain for this host. + /// + /// # Errors + /// + /// Returns [`BuildError::AndroidNdkUnsupportedHostPlatform`] for a host the NDK ships no + /// toolchain for. + pub const fn android_ndk_tag(self) -> Result<&'static str, BuildError> { + match self { + Self::Linux => Ok("linux-x86_64"), + Self::MacOs => Ok("darwin-x86_64"), + Self::Unsupported => Err(BuildError::AndroidNdkUnsupportedHostPlatform), + } + } +} + +#[cfg(test)] +mod tests { + use super::HostOs; + + #[test] + fn hosts_are_named_like_the_standard_library_names_them() { + assert_eq!( + ["linux", "macos", "windows"].map(HostOs::named), + [HostOs::Linux, HostOs::MacOs, HostOs::Unsupported] + ); + } + + #[test] + fn every_supported_host_names_its_prebuilt_ndk_toolchain() { + assert_eq!( + [HostOs::Linux, HostOs::MacOs].map(|host_os| host_os.android_ndk_tag().ok()), + [Some("linux-x86_64"), Some("darwin-x86_64")] + ); + } + + #[test] + fn an_unsupported_host_has_no_ndk_toolchain() { + assert_eq!( + HostOs::Unsupported + .android_ndk_tag() + .err() + .map(|error| error.to_string()), + Some("Unsupported host platform for Android NDK".to_owned()) + ); + } +} diff --git a/llama-cpp-bindings-build/src/lib.rs b/llama-cpp-bindings-build/src/lib.rs index a690ab144..a23bb974d 100644 --- a/llama-cpp-bindings-build/src/lib.rs +++ b/llama-cpp-bindings-build/src/lib.rs @@ -2,191 +2,140 @@ not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] + +mod android_architecture; mod android_ndk; -mod apple_variant; -mod bindgen_config; -mod cmake_config; -mod cpp_wrapper; -mod library_linking; +mod binding_callbacks; +mod bindings_generation; +mod build_context; +pub mod build_environment; +pub mod build_error; +pub mod build_feature; +mod cargo_directive; +mod cmake_build; +mod cmake_cache; +mod cmake_cache_entry; +mod cmake_configuration; +mod cmake_define; +mod cmake_file_api; +mod cmake_replies; +mod cmake_reply_file; +mod cmake_reply_index; +mod cpu_tuning; +mod emit_directives; +mod enabled_build_features; +mod host_os; +mod link_directives; +mod link_kind; +mod native_build; mod native_library; -mod rebuild_tracking; +mod native_profile; +mod openmp_linking; +mod openmp_runtime_source; +mod privatized_field; +mod rebuild_triggers; mod target_os; +mod wrapper_compilation; +mod x86_ggml_switch; -use std::env; -use std::path::{Path, PathBuf}; - -use android_ndk::AndroidNdk; -use target_os::TargetOs; - -#[derive(Debug, thiserror::Error)] -pub enum BuildError { - #[error("environment variable {name} could not be read: {source}")] - Environment { - name: &'static str, - #[source] - source: env::VarError, - }, - #[error("unsupported target operating system: {cargo_cfg_target_os}")] - UnsupportedTargetOs { cargo_cfg_target_os: String }, - #[error(transparent)] - AndroidNdk(#[from] android_ndk::AndroidNdkDetectionError), - #[error("bindgen failed: {0}")] - Bindgen(#[source] bindgen::BindgenError), - #[error( - "bindgen never saw field {type_name}::{field_name}, so it can no longer be kept private" - )] - PrivatizedFieldMissing { - type_name: &'static str, - field_name: &'static str, - }, - #[error("generated bindings could not be written: {0}")] - BindingsWrite(#[source] std::io::Error), - #[error("native wrapper compilation failed: {0}")] - NativeWrapper(#[source] cc::Error), - #[error("filesystem operation failed for {path}: {source}")] - Filesystem { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("environment path {name} does not name a directory: {path}")] - EnvironmentDirectory { name: &'static str, path: PathBuf }, -} - -fn required_env(name: &'static str) -> Result { - env::var(name).map_err(|source| BuildError::Environment { name, source }) -} - -fn optional_env(name: &'static str) -> Result, BuildError> { - match env::var(name) { - Ok(value) => Ok(Some(value)), - Err(env::VarError::NotPresent) => Ok(None), - Err(source) => Err(BuildError::Environment { name, source }), - } -} - -#[macro_export] -macro_rules! debug_log { - ($($arg:tt)*) => { - if std::env::var("BUILD_DEBUG").is_ok() { - println!("cargo:warning=[DEBUG] {}", format!($($arg)*)); - } - }; -} - -#[derive(Debug)] -pub struct BuildContext { - pub out_dir: PathBuf, - pub cmake_dir: PathBuf, - pub llama_src: PathBuf, - pub target_os: TargetOs, - pub target_triple: String, - pub cargo_cfg_target_arch: String, - pub cargo_cfg_target_env: String, - pub build_shared_libs: bool, - pub profile: String, - pub android_ndk: Option, -} - -impl BuildContext { - fn detect() -> Result { - let target_triple = required_env("TARGET")?; - let cargo_cfg_target_os = required_env("CARGO_CFG_TARGET_OS")?; - let cargo_cfg_target_env = optional_env("CARGO_CFG_TARGET_ENV")?.unwrap_or_default(); - let target_os = TargetOs::from_cargo_cfg(&cargo_cfg_target_os).ok_or_else(|| { - BuildError::UnsupportedTargetOs { - cargo_cfg_target_os: cargo_cfg_target_os.clone(), - } - })?; - let out_dir = PathBuf::from(required_env("OUT_DIR")?); - let manifest_dir = required_env("CARGO_MANIFEST_DIR")?; - let llama_src = Path::new(&manifest_dir).join("llama.cpp"); - - let build_shared_libs = cfg!(feature = "dynamic-link"); - let profile = native_profile(&required_env("PROFILE")?); +use std::io::Write; - let cargo_cfg_target_arch = required_env("CARGO_CFG_TARGET_ARCH")?; - let android_ndk = if target_os.is_android() { - Some(AndroidNdk::detect(&target_triple, &cargo_cfg_target_arch)?) - } else { - None - }; - - let cmake_dir = out_dir.join("cmake"); - - debug_log!("TARGET: {}", target_triple); - debug_log!("CARGO_MANIFEST_DIR: {}", manifest_dir); - debug_log!("OUT_DIR: {}", out_dir.display()); - debug_log!("CMAKE_DIR: {}", cmake_dir.display()); - debug_log!("BUILD_SHARED: {}", build_shared_libs); - - Ok(Self { - out_dir, - cmake_dir, - llama_src, - target_os, - target_triple, - cargo_cfg_target_arch, - cargo_cfg_target_env, - build_shared_libs, - profile, - android_ndk, - }) - } -} - -fn native_profile(cargo_profile: &str) -> String { - match cargo_profile { - "debug" | "release" => "Release".to_owned(), - other => other.to_owned(), - } -} +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::host_os::HostOs; +use crate::native_build::NativeBuild; +/// Generates the bindings, compiles the C++ wrappers, and builds llama.cpp for the build script +/// run `environment` describes, then writes the cargo directives that link them to `output`. +/// /// # Errors /// -/// Returns [`BuildError`] when the build environment cannot be read, the target is -/// unsupported, or any of the native build steps fail. -pub fn build() -> Result<(), BuildError> { - let context = BuildContext::detect()?; +/// Returns the [`BuildError`] of the first decision or build step that fails; nothing is written +/// to `output` then. +pub fn build(environment: &BuildEnvironment, output: &mut impl Write) -> Result<(), BuildError> { + let context = BuildContext::from_environment(environment, HostOs::current())?; - rebuild_tracking::register_rebuild_triggers(&context.llama_src); + NativeBuild::of(&context, environment)?.run(output) +} - bindgen_config::generate_bindings( - &context.llama_src, - &context.out_dir, - context.target_os, - &context.target_triple, - context.android_ndk.as_ref(), - )?; +#[cfg(test)] +mod tests { + use std::ffi::OsStr; - cpp_wrapper::compile_cpp_wrappers(context.target_os)?; + use enumflags2::BitFlags; - let build_dir = cmake_config::configure_and_build(&context)?; + use super::build; + use crate::build_context::tests::cargo_environment; + use crate::build_environment::BuildEnvironment; + use crate::build_environment::tests::variable; + use crate::build_feature::BuildFeature; - library_linking::link_libraries( - &context.cmake_dir, - &build_dir, - context.target_os, - &context.cargo_cfg_target_env, - context.build_shared_libs, - &context.profile, - )?; + const FIXTURE_CMAKE_PROJECT: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/cmake_project"); - Ok(()) -} + fn failure(environment: &BuildEnvironment) -> [Option; 2] { + let mut output = Vec::new(); -#[cfg(test)] -mod build_context_value_tests { - use super::native_profile; - - #[test] - fn cargo_debug_and_release_profiles_both_use_optimized_native_code() { - assert_eq!(native_profile("debug"), "Release"); - assert_eq!(native_profile("release"), "Release"); + [ + build(environment, &mut output) + .err() + .map(|error| error.to_string()), + String::from_utf8(output).ok(), + ] } #[test] - fn custom_cargo_profile_name_is_preserved_for_cmake() { - assert_eq!(native_profile("RelWithDebInfo"), "RelWithDebInfo"); + fn a_failure_at_any_stage_is_reported_and_nothing_is_emitted() { + let mut without_target = cargo_environment(BitFlags::empty(), &[]); + + without_target.variables.remove(OsStr::new("TARGET")); + + assert_eq!( + [ + failure(&without_target), + failure(&cargo_environment( + BitFlags::from(BuildFeature::SystemGgml), + &[] + )), + failure(&cargo_environment(BitFlags::from(BuildFeature::Rocm), &[])), + failure(&cargo_environment( + BitFlags::empty(), + &[variable("CARGO_MANIFEST_DIR", FIXTURE_CMAKE_PROJECT)] + )), + ], + [ + [ + Some( + "environment variable TARGET could not be read: environment variable not \ + found" + .to_owned() + ), + Some(String::new()) + ], + [ + Some( + "environment variable GGML_DIR could not be read: environment variable \ + not found" + .to_owned() + ), + Some(String::new()) + ], + [ + Some( + "environment variable ROCM_PATH could not be read: environment variable \ + not found" + .to_owned() + ), + Some(String::new()) + ], + [ + Some(format!( + "bindgen failed: header '{FIXTURE_CMAKE_PROJECT}/wrapper.h' does not exist." + )), + Some(String::new()) + ], + ] + ); } } diff --git a/llama-cpp-bindings-build/src/library_linking.rs b/llama-cpp-bindings-build/src/library_linking.rs deleted file mode 100644 index c779bfc3d..000000000 --- a/llama-cpp-bindings-build/src/library_linking.rs +++ /dev/null @@ -1,381 +0,0 @@ -use std::env; -use std::path::Path; - -use crate::BuildError; -use crate::apple_variant::AppleVariant; -use crate::debug_log; -use crate::native_library::NativeLibrary; -use crate::target_os::TargetOs; - -pub fn link_libraries( - cmake_dir: &Path, - build_dir: &Path, - target_os: TargetOs, - cargo_cfg_target_env: &str, - build_shared_libs: bool, - profile: &str, -) -> Result<(), BuildError> { - emit_search_paths(cmake_dir, build_dir); - link_system_ggml_paths()?; - link_cmake_built_libraries(cmake_dir, build_shared_libs, profile); - link_cuda_libraries(build_shared_libs); - link_rocm_libraries(build_shared_libs)?; - link_openmp(cargo_cfg_target_env); - link_platform_system_libraries(target_os); - - Ok(()) -} - -fn emit_search_paths(cmake_dir: &Path, build_dir: &Path) { - println!( - "cargo:rustc-link-search={}", - cmake_dir.join("lib").display() - ); - println!( - "cargo:rustc-link-search={}", - cmake_dir.join("lib64").display() - ); - println!("cargo:rustc-link-search={}", build_dir.display()); -} - -fn link_system_ggml_paths() -> Result<(), BuildError> { - if !cfg!(feature = "system-ggml") { - return Ok(()); - } - - println!("cargo:rerun-if-env-changed=GGML_LIBRARY_DIR"); - let library_dir = env::var("GGML_LIBRARY_DIR").map_err(|source| BuildError::Environment { - name: "GGML_LIBRARY_DIR", - source, - })?; - let library_dir = Path::new(&library_dir); - if !library_dir.is_dir() { - return Err(BuildError::EnvironmentDirectory { - name: "GGML_LIBRARY_DIR", - path: library_dir.to_path_buf(), - }); - } - println!("cargo:rustc-link-search=native={}", library_dir.display()); - - Ok(()) -} - -fn link_cmake_built_libraries(cmake_dir: &Path, build_shared_libs: bool, profile: &str) { - emit_private_dependency_search_paths(cmake_dir, profile); - for NativeLibrary { link_kind, name } in native_libraries(build_shared_libs) { - let link = format!("cargo:rustc-link-lib={link_kind}={name}"); - debug_log!("LINK {link}"); - println!("{link}"); - } -} - -fn native_libraries(build_shared_libs: bool) -> Vec { - let cmake_kind = if build_shared_libs { "dylib" } else { "static" }; - let mut libraries = vec![NativeLibrary { - link_kind: cmake_kind, - name: "llama-common", - }]; - - if !build_shared_libs { - libraries.extend([ - NativeLibrary { - link_kind: "static", - name: "llama-common-base", - }, - NativeLibrary { - link_kind: "static", - name: "cpp-httplib", - }, - ]); - } - - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "mtmd", - }); - if !build_shared_libs { - libraries.push(NativeLibrary { - link_kind: "static", - name: "vendor-hash", - }); - } - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "llama", - }); - - if cfg!(feature = "system-ggml") { - let ggml_kind = if cfg!(feature = "system-ggml-static") { - "static" - } else { - "dylib" - }; - libraries.extend([ - NativeLibrary { - link_kind: ggml_kind, - name: "ggml-cpu", - }, - NativeLibrary { - link_kind: ggml_kind, - name: "ggml-base", - }, - NativeLibrary { - link_kind: ggml_kind, - name: "ggml", - }, - ]); - return libraries; - } - - if !cfg!(feature = "dynamic-backends") { - if cfg!(feature = "cuda") { - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "ggml-cuda", - }); - } - if cfg!(feature = "metal") { - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "ggml-metal", - }); - } - if cfg!(feature = "vulkan") { - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "ggml-vulkan", - }); - } - if cfg!(feature = "rocm") { - libraries.push(NativeLibrary { - link_kind: cmake_kind, - name: "ggml-hip", - }); - } - } - - libraries.extend([ - NativeLibrary { - link_kind: cmake_kind, - name: "ggml-cpu", - }, - NativeLibrary { - link_kind: cmake_kind, - name: "ggml-base", - }, - NativeLibrary { - link_kind: cmake_kind, - name: "ggml", - }, - ]); - libraries -} - -fn emit_private_dependency_search_paths(cmake_dir: &Path, profile: &str) { - let common_lib_dir = cmake_dir.join("build").join("common"); - let httplib_dir = cmake_dir.join("build").join("vendor").join("cpp-httplib"); - let hash_dir = cmake_dir.join("build").join("vendor").join("hash"); - - emit_search_path_with_profile(&common_lib_dir, profile); - emit_search_path_with_profile(&httplib_dir, profile); - emit_search_path_with_profile(&hash_dir, profile); -} - -fn emit_search_path_with_profile(lib_dir: &Path, profile: &str) { - println!("cargo:rustc-link-search=native={}", lib_dir.display()); - - let profile_dir = lib_dir.join(profile); - - println!("cargo:rustc-link-search=native={}", profile_dir.display()); -} - -fn link_cuda_libraries(build_shared_libs: bool) { - if !cfg!(feature = "cuda") || build_shared_libs { - return; - } - - println!("cargo:rerun-if-env-changed=CUDA_PATH"); - - for lib_dir in find_cuda_helper::find_cuda_lib_dirs() { - println!("cargo:rustc-link-search=native={}", lib_dir.display()); - } - - println!("cargo:rustc-link-lib=static=cudart_static"); - println!("cargo:rustc-link-lib=static=cublas_static"); - println!("cargo:rustc-link-lib=static=cublasLt_static"); - - if !cfg!(feature = "cuda-no-vmm") { - println!("cargo:rustc-link-lib=cuda"); - } - - println!("cargo:rustc-link-lib=static=culibos"); -} - -fn link_rocm_libraries(build_shared_libs: bool) -> Result<(), BuildError> { - if !cfg!(feature = "rocm") || build_shared_libs { - return Ok(()); - } - - println!("cargo:rerun-if-env-changed=ROCM_PATH"); - let rocm_path = env::var("ROCM_PATH").map_err(|source| BuildError::Environment { - name: "ROCM_PATH", - source, - })?; - - let rocm_lib = Path::new(&rocm_path).join("lib"); - - if !rocm_lib.is_dir() { - return Err(BuildError::EnvironmentDirectory { - name: "ROCM_PATH", - path: rocm_lib, - }); - } - - println!("cargo:rustc-link-search=native={}", rocm_lib.display()); - println!("cargo:rustc-link-lib=dylib=amdhip64"); - println!("cargo:rustc-link-lib=dylib=rocblas"); - println!("cargo:rustc-link-lib=dylib=hipblas"); - - Ok(()) -} - -fn link_openmp(cargo_cfg_target_env: &str) { - if cfg!(feature = "openmp") && cargo_cfg_target_env == "gnu" { - println!("cargo:rustc-link-lib=gomp"); - } -} - -fn link_platform_system_libraries(target_os: TargetOs) { - match target_os { - TargetOs::Linux => { - println!("cargo:rustc-link-lib=dylib=stdc++"); - } - TargetOs::Apple(variant) => { - link_apple_frameworks(variant); - } - TargetOs::Android => { - link_android_cpp_stdlib(); - } - } -} - -fn link_android_cpp_stdlib() { - if cfg!(feature = "static-stdcxx") { - println!("cargo:rustc-link-lib=c++_static"); - println!("cargo:rustc-link-lib=c++abi"); - } else if cfg!(feature = "shared-stdcxx") { - println!("cargo:rustc-link-lib=c++_shared"); - } -} - -fn link_apple_frameworks(_variant: AppleVariant) { - println!("cargo:rustc-link-lib=framework=Foundation"); - println!("cargo:rustc-link-lib=framework=Metal"); - println!("cargo:rustc-link-lib=framework=MetalKit"); - println!("cargo:rustc-link-lib=framework=Accelerate"); - println!("cargo:rustc-link-lib=c++"); -} - -#[cfg(test)] -mod native_link_graph_tests { - #[cfg(feature = "cuda")] - use super::NativeLibrary; - use super::native_libraries; - - #[test] - fn the_cuda_backend_archive_is_linked_only_when_the_feature_is_enabled() { - let links_cuda_backend = native_libraries(false) - .iter() - .any(|library| library.name == "ggml-cuda"); - - assert_eq!(links_cuda_backend, cfg!(feature = "cuda")); - } - - #[cfg(feature = "cuda")] - #[test] - fn static_cuda_link_graph_contains_every_owned_archive_in_dependency_order() { - assert_eq!( - native_libraries(false), - vec![ - NativeLibrary { - link_kind: "static", - name: "llama-common", - }, - NativeLibrary { - link_kind: "static", - name: "llama-common-base", - }, - NativeLibrary { - link_kind: "static", - name: "cpp-httplib", - }, - NativeLibrary { - link_kind: "static", - name: "mtmd", - }, - NativeLibrary { - link_kind: "static", - name: "vendor-hash", - }, - NativeLibrary { - link_kind: "static", - name: "llama", - }, - NativeLibrary { - link_kind: "static", - name: "ggml-cuda", - }, - NativeLibrary { - link_kind: "static", - name: "ggml-cpu", - }, - NativeLibrary { - link_kind: "static", - name: "ggml-base", - }, - NativeLibrary { - link_kind: "static", - name: "ggml", - }, - ] - ); - } - - #[cfg(feature = "cuda")] - #[test] - fn dynamic_cuda_link_graph_uses_shared_top_level_libraries() { - assert_eq!( - native_libraries(true), - vec![ - NativeLibrary { - link_kind: "dylib", - name: "llama-common", - }, - NativeLibrary { - link_kind: "dylib", - name: "mtmd", - }, - NativeLibrary { - link_kind: "dylib", - name: "llama", - }, - NativeLibrary { - link_kind: "dylib", - name: "ggml-cuda", - }, - NativeLibrary { - link_kind: "dylib", - name: "ggml-cpu", - }, - NativeLibrary { - link_kind: "dylib", - name: "ggml-base", - }, - NativeLibrary { - link_kind: "dylib", - name: "ggml", - }, - ] - ); - } -} diff --git a/llama-cpp-bindings-build/src/link_directives.rs b/llama-cpp-bindings-build/src/link_directives.rs new file mode 100644 index 000000000..5b08ee3db --- /dev/null +++ b/llama-cpp-bindings-build/src/link_directives.rs @@ -0,0 +1,659 @@ +use std::path::Path; +use std::path::PathBuf; + +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cargo_directive::CargoDirective; +use crate::link_kind::LinkKind; +use crate::native_library::NativeLibrary; +use crate::native_profile::NATIVE_PROFILE; +use crate::target_os::TargetOs; + +const fn link(link_kind: LinkKind, name: &'static str) -> CargoDirective { + CargoDirective::LinkLibrary(NativeLibrary { link_kind, name }) +} + +fn existing_directory(name: &'static str, path: PathBuf) -> Result { + if path.is_dir() { + Ok(path) + } else { + Err(BuildError::EnvironmentDirectory { name, path }) + } +} + +fn cmake_search_paths(context: &BuildContext) -> Vec { + let cmake_dir = context.cmake_dir(); + + vec![ + CargoDirective::LinkSearch(cmake_dir.join("lib")), + CargoDirective::LinkSearch(cmake_dir.join("lib64")), + CargoDirective::LinkSearch(cmake_dir), + ] +} + +fn system_ggml_search_paths( + environment: &BuildEnvironment, +) -> Result, BuildError> { + if !environment.has(BuildFeature::SystemGgml) { + return Ok(Vec::new()); + } + + let library_dir = existing_directory( + "GGML_LIBRARY_DIR", + PathBuf::from(environment.required("GGML_LIBRARY_DIR")?), + )?; + + Ok(vec![ + CargoDirective::RerunIfEnvChanged("GGML_LIBRARY_DIR".to_owned()), + CargoDirective::LinkSearchNative(library_dir), + ]) +} + +fn private_dependency_search_paths(context: &BuildContext) -> Vec { + let build_dir = context.cmake_build_dir(); + + [ + build_dir.join("common"), + build_dir.join("vendor").join("cpp-httplib"), + build_dir.join("vendor").join("hash"), + ] + .into_iter() + .flat_map(|library_dir| { + [ + CargoDirective::LinkSearchNative(library_dir.clone()), + CargoDirective::LinkSearchNative(library_dir.join(NATIVE_PROFILE)), + ] + }) + .collect() +} + +fn native_libraries(context: &BuildContext, environment: &BuildEnvironment) -> Vec { + let cmake_kind = if context.build_shared_libs { + LinkKind::Dylib + } else { + LinkKind::Static + }; + let built = |name| NativeLibrary { + link_kind: cmake_kind, + name, + }; + let bundled = |name| NativeLibrary { + link_kind: LinkKind::Static, + name, + }; + let mut libraries = vec![built("llama-common")]; + + if !context.build_shared_libs { + libraries.extend([bundled("llama-common-base"), bundled("cpp-httplib")]); + } + + libraries.push(built("mtmd")); + + if !context.build_shared_libs { + libraries.push(bundled("vendor-hash")); + } + + libraries.push(built("llama")); + + if environment.has(BuildFeature::SystemGgml) { + let ggml_kind = if environment.has(BuildFeature::SystemGgmlStatic) { + LinkKind::Static + } else { + LinkKind::Dylib + }; + + libraries.extend(["ggml-cpu", "ggml-base", "ggml"].map(|name| NativeLibrary { + link_kind: ggml_kind, + name, + })); + + return libraries; + } + + if !environment.has(BuildFeature::DynamicBackends) { + if environment.has(BuildFeature::Cuda) { + libraries.push(built("ggml-cuda")); + } + + if environment.has(BuildFeature::Metal) { + libraries.push(built("ggml-metal")); + } + + if environment.has(BuildFeature::Vulkan) { + libraries.push(built("ggml-vulkan")); + } + + if environment.has(BuildFeature::Rocm) { + libraries.push(built("ggml-hip")); + } + + libraries.push(built("ggml-cpu")); + } + + libraries.extend(["ggml-base", "ggml"].map(built)); + + libraries +} + +fn native_library_links( + context: &BuildContext, + environment: &BuildEnvironment, +) -> Vec { + native_libraries(context, environment) + .into_iter() + .flat_map(|library| { + let directive = CargoDirective::LinkLibrary(library); + + [ + CargoDirective::Debug(format!("LINK {directive}")), + directive, + ] + }) + .collect() +} + +fn cuda_links(context: &BuildContext, environment: &BuildEnvironment) -> Vec { + if !environment.has(BuildFeature::Cuda) || context.build_shared_libs { + return Vec::new(); + } + + let mut directives = vec![ + CargoDirective::RerunIfEnvChanged("CUDA_PATH".to_owned()), + CargoDirective::RerunIfEnvChanged("CUDA_LIBRARY_PATH".to_owned()), + ]; + + directives.extend( + environment + .cuda_library_dirs + .iter() + .cloned() + .map(CargoDirective::LinkSearchNative), + ); + directives.extend( + ["cudart_static", "cublas_static", "cublasLt_static"] + .map(|name| link(LinkKind::Static, name)), + ); + + if !environment.has(BuildFeature::CudaNoVmm) { + directives.push(link(LinkKind::Unspecified, "cuda")); + } + + directives.push(link(LinkKind::Static, "culibos")); + + directives +} + +fn rocm_links( + context: &BuildContext, + environment: &BuildEnvironment, +) -> Result, BuildError> { + if !environment.has(BuildFeature::Rocm) || context.build_shared_libs { + return Ok(Vec::new()); + } + + let rocm_lib = existing_directory( + "ROCM_PATH", + Path::new(&environment.required("ROCM_PATH")?).join("lib"), + )?; + + Ok(vec![ + CargoDirective::RerunIfEnvChanged("ROCM_PATH".to_owned()), + CargoDirective::LinkSearchNative(rocm_lib), + link(LinkKind::Dylib, "amdhip64"), + link(LinkKind::Dylib, "rocblas"), + link(LinkKind::Dylib, "hipblas"), + ]) +} + +fn platform_links(context: &BuildContext, environment: &BuildEnvironment) -> Vec { + match context.target_os { + TargetOs::Linux => vec![link(LinkKind::Dylib, "stdc++")], + TargetOs::Apple if environment.has(BuildFeature::Metal) => vec![ + link(LinkKind::Framework, "Foundation"), + link(LinkKind::Framework, "Metal"), + link(LinkKind::Framework, "MetalKit"), + link(LinkKind::Framework, "Accelerate"), + link(LinkKind::Unspecified, "c++"), + ], + TargetOs::Apple => vec![ + link(LinkKind::Framework, "Accelerate"), + link(LinkKind::Unspecified, "c++"), + ], + TargetOs::Android(_) if environment.has(BuildFeature::StaticStdcxx) => vec![ + link(LinkKind::Unspecified, "c++_static"), + link(LinkKind::Unspecified, "c++abi"), + ], + TargetOs::Android(_) if environment.has(BuildFeature::SharedStdcxx) => { + vec![link(LinkKind::Unspecified, "c++_shared")] + } + TargetOs::Android(_) => Vec::new(), + } +} + +/// The search paths and libraries that link the native build into the bindings, in the order +/// the linker resolves them. +/// +/// # Errors +/// +/// Returns [`BuildError::Environment`] when a library directory an enabled feature needs is not +/// set, or [`BuildError::EnvironmentDirectory`] when it does not name a directory. +pub fn link_directives( + context: &BuildContext, + environment: &BuildEnvironment, +) -> Result, BuildError> { + Ok([ + cmake_search_paths(context), + system_ggml_search_paths(environment)?, + private_dependency_search_paths(context), + native_library_links(context, environment), + cuda_links(context, environment), + rocm_links(context, environment)?, + platform_links(context, environment), + ] + .concat()) +} + +#[cfg(test)] +mod tests { + use std::env; + use std::fs; + use std::path::Path; + use std::path::PathBuf; + use std::process; + + use enumflags2::BitFlags; + + use super::link_directives; + use crate::build_context::tests::android_overrides; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_environment::BuildEnvironment; + use crate::build_environment::tests::EnvironmentVariable; + use crate::build_environment::tests::variable; + use crate::build_feature::BuildFeature; + use crate::cargo_directive::CargoDirective; + use crate::link_kind::LinkKind; + use crate::native_library::NativeLibrary; + + const FIXTURES: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures"); + + fn linked(environment: &BuildEnvironment) -> Vec { + link_directives(&context_of(environment), environment).unwrap() + } + + fn libraries(directives: &[CargoDirective]) -> Vec { + directives + .iter() + .filter_map(|directive| match directive { + CargoDirective::LinkLibrary(library) => Some(*library), + _ => None, + }) + .collect() + } + + fn search_paths_and_rebuild_triggers(directives: &[CargoDirective]) -> Vec { + directives + .iter() + .filter(|directive| { + !directive.is_debug() && !matches!(directive, CargoDirective::LinkLibrary(_)) + }) + .cloned() + .collect() + } + + fn platform_libraries( + features: BitFlags, + overrides: &[EnvironmentVariable], + ) -> Vec { + let libraries = libraries(&linked(&cargo_environment(features, overrides))); + + libraries + .split(|library| library.name == "ggml") + .next_back() + .unwrap() + .to_vec() + } + + fn library(link_kind: LinkKind, name: &'static str) -> NativeLibrary { + NativeLibrary { link_kind, name } + } + + fn cmake_search_paths() -> Vec { + vec![ + CargoDirective::LinkSearch(PathBuf::from("/out/cmake/lib")), + CargoDirective::LinkSearch(PathBuf::from("/out/cmake/lib64")), + CargoDirective::LinkSearch(PathBuf::from("/out/cmake")), + ] + } + + fn private_dependency_search_paths() -> Vec { + [ + "/out/cmake/build/common", + "/out/cmake/build/common/Release", + "/out/cmake/build/vendor/cpp-httplib", + "/out/cmake/build/vendor/cpp-httplib/Release", + "/out/cmake/build/vendor/hash", + "/out/cmake/build/vendor/hash/Release", + ] + .map(|path| CargoDirective::LinkSearchNative(PathBuf::from(path))) + .to_vec() + } + + fn static_llama_libraries() -> Vec { + [ + "llama-common", + "llama-common-base", + "cpp-httplib", + "mtmd", + "vendor-hash", + "llama", + ] + .map(|name| library(LinkKind::Static, name)) + .to_vec() + } + + #[test] + fn a_static_linux_build_links_every_archive_after_its_dependents_and_then_libstdcxx() { + let static_archives = [ + static_llama_libraries(), + ["ggml-cpu", "ggml-base", "ggml"] + .map(|name| library(LinkKind::Static, name)) + .to_vec(), + ] + .concat() + .into_iter() + .flat_map(|archive| { + let directive = CargoDirective::LinkLibrary(archive); + + [ + CargoDirective::Debug(format!("LINK {directive}")), + directive, + ] + }) + .collect::>(); + + assert_eq!( + linked(&cargo_environment(BitFlags::empty(), &[])), + [ + cmake_search_paths(), + private_dependency_search_paths(), + static_archives, + vec![CargoDirective::LinkLibrary(library( + LinkKind::Dylib, + "stdc++" + ))], + ] + .concat() + ); + } + + #[test] + fn a_dynamic_backends_build_links_shared_libraries_and_leaves_every_backend_to_runtime_loading() + { + assert_eq!( + libraries(&linked(&cargo_environment( + BuildFeature::DynamicLink + | BuildFeature::DynamicBackends + | BuildFeature::Cuda + | BuildFeature::Rocm, + &[] + ))), + [ + "llama-common", + "mtmd", + "llama", + "ggml-base", + "ggml", + "stdc++" + ] + .map(|name| library(LinkKind::Dylib, name)) + ); + } + + #[test] + fn static_gpu_backends_link_their_archives_and_the_vendor_runtimes() { + let rocm_root = + env::temp_dir().join(format!("llama-cpp-bindings-build-rocm-{}", process::id())); + fs::create_dir_all(rocm_root.join("lib")).unwrap(); + let environment = BuildEnvironment { + cuda_library_dirs: vec![PathBuf::from("/cuda/lib64")], + ..cargo_environment( + BuildFeature::Cuda + | BuildFeature::Metal + | BuildFeature::Vulkan + | BuildFeature::Rocm, + &[variable("ROCM_PATH", rocm_root.to_str().unwrap())], + ) + }; + let directives = linked(&environment); + + assert_eq!( + libraries(&directives), + [ + static_llama_libraries(), + [ + "ggml-cuda", + "ggml-metal", + "ggml-vulkan", + "ggml-hip", + "ggml-cpu", + "ggml-base", + "ggml", + "cudart_static", + "cublas_static", + "cublasLt_static", + ] + .map(|name| library(LinkKind::Static, name)) + .to_vec(), + vec![ + library(LinkKind::Unspecified, "cuda"), + library(LinkKind::Static, "culibos"), + library(LinkKind::Dylib, "amdhip64"), + library(LinkKind::Dylib, "rocblas"), + library(LinkKind::Dylib, "hipblas"), + library(LinkKind::Dylib, "stdc++"), + ], + ] + .concat() + ); + assert_eq!( + search_paths_and_rebuild_triggers(&directives), + [ + cmake_search_paths(), + private_dependency_search_paths(), + vec![ + CargoDirective::RerunIfEnvChanged("CUDA_PATH".to_owned()), + CargoDirective::RerunIfEnvChanged("CUDA_LIBRARY_PATH".to_owned()), + CargoDirective::LinkSearchNative(PathBuf::from("/cuda/lib64")), + CargoDirective::RerunIfEnvChanged("ROCM_PATH".to_owned()), + CargoDirective::LinkSearchNative(rocm_root.join("lib")), + ], + ] + .concat() + ); + fs::remove_dir_all(rocm_root).unwrap(); + } + + #[test] + fn cuda_without_virtual_memory_management_does_not_link_the_driver() { + assert_eq!( + libraries(&linked(&cargo_environment( + BuildFeature::Cuda | BuildFeature::CudaNoVmm, + &[] + ))), + [ + static_llama_libraries(), + [ + "ggml-cuda", + "ggml-cpu", + "ggml-base", + "ggml", + "cudart_static", + "cublas_static", + "cublasLt_static", + "culibos", + ] + .map(|name| library(LinkKind::Static, name)) + .to_vec(), + vec![library(LinkKind::Dylib, "stdc++")], + ] + .concat() + ); + } + + #[test] + fn system_ggml_links_the_installed_ggml_instead_of_any_built_backend() { + let system_ggml_linked = |features: BitFlags| { + linked(&cargo_environment( + features | BuildFeature::SystemGgml | BuildFeature::Cuda, + &[variable("GGML_LIBRARY_DIR", FIXTURES)], + )) + }; + let linked_with_ggml_as = |ggml_kind: LinkKind| { + [ + static_llama_libraries(), + ["ggml-cpu", "ggml-base", "ggml"] + .map(|name| library(ggml_kind, name)) + .to_vec(), + ["cudart_static", "cublas_static", "cublasLt_static"] + .map(|name| library(LinkKind::Static, name)) + .to_vec(), + vec![ + library(LinkKind::Unspecified, "cuda"), + library(LinkKind::Static, "culibos"), + library(LinkKind::Dylib, "stdc++"), + ], + ] + .concat() + }; + let shared_ggml = system_ggml_linked(BitFlags::empty()); + + assert_eq!( + [ + libraries(&shared_ggml), + libraries(&system_ggml_linked(BitFlags::from( + BuildFeature::SystemGgmlStatic + ))), + ], + [ + linked_with_ggml_as(LinkKind::Dylib), + linked_with_ggml_as(LinkKind::Static) + ] + ); + assert_eq!( + search_paths_and_rebuild_triggers(&shared_ggml), + [ + cmake_search_paths(), + vec![ + CargoDirective::RerunIfEnvChanged("GGML_LIBRARY_DIR".to_owned()), + CargoDirective::LinkSearchNative(PathBuf::from(FIXTURES)), + ], + private_dependency_search_paths(), + vec![ + CargoDirective::RerunIfEnvChanged("CUDA_PATH".to_owned()), + CargoDirective::RerunIfEnvChanged("CUDA_LIBRARY_PATH".to_owned()), + ], + ] + .concat() + ); + } + + #[test] + fn apple_and_android_targets_link_their_platform_cxx_runtime_and_metal_only_with_metal() { + assert_eq!( + [ + platform_libraries( + BitFlags::from(BuildFeature::Metal), + &[variable("CARGO_CFG_TARGET_OS", "macos")] + ), + platform_libraries( + BitFlags::empty(), + &[variable("CARGO_CFG_TARGET_OS", "macos")] + ), + platform_libraries( + BitFlags::from(BuildFeature::StaticStdcxx), + &android_overrides() + ), + platform_libraries( + BitFlags::from(BuildFeature::SharedStdcxx), + &android_overrides() + ), + platform_libraries(BitFlags::empty(), &android_overrides()), + ], + [ + vec![ + library(LinkKind::Framework, "Foundation"), + library(LinkKind::Framework, "Metal"), + library(LinkKind::Framework, "MetalKit"), + library(LinkKind::Framework, "Accelerate"), + library(LinkKind::Unspecified, "c++"), + ], + vec![ + library(LinkKind::Framework, "Accelerate"), + library(LinkKind::Unspecified, "c++"), + ], + vec![ + library(LinkKind::Unspecified, "c++_static"), + library(LinkKind::Unspecified, "c++abi"), + ], + vec![library(LinkKind::Unspecified, "c++_shared")], + Vec::new(), + ] + ); + } + + #[test] + fn library_directories_an_enabled_feature_needs_must_be_set_and_exist() { + let missing_directory = env::temp_dir().join(format!( + "llama-cpp-bindings-build-missing-{}", + process::id() + )); + let refusal = |features: BitFlags, overrides: &[EnvironmentVariable]| { + let environment = cargo_environment(features, overrides); + + link_directives(&context_of(&environment), &environment) + .err() + .map(|error| error.to_string()) + }; + + assert_eq!( + [ + refusal(BitFlags::from(BuildFeature::SystemGgml), &[]), + refusal( + BitFlags::from(BuildFeature::SystemGgml), + &[variable( + "GGML_LIBRARY_DIR", + missing_directory.to_str().unwrap() + )] + ), + refusal(BitFlags::from(BuildFeature::Rocm), &[]), + refusal( + BitFlags::from(BuildFeature::Rocm), + &[variable("ROCM_PATH", env!("CARGO_MANIFEST_DIR"))] + ), + ], + [ + Some( + "environment variable GGML_LIBRARY_DIR could not be read: environment \ + variable not found" + .to_owned() + ), + Some(format!( + "environment path GGML_LIBRARY_DIR does not name a directory: {}", + missing_directory.display() + )), + Some( + "environment variable ROCM_PATH could not be read: environment variable not \ + found" + .to_owned() + ), + Some(format!( + "environment path ROCM_PATH does not name a directory: {}", + Path::new(env!("CARGO_MANIFEST_DIR")).join("lib").display() + )), + ] + ); + } +} diff --git a/llama-cpp-bindings-build/src/link_kind.rs b/llama-cpp-bindings-build/src/link_kind.rs new file mode 100644 index 000000000..b3d6b4beb --- /dev/null +++ b/llama-cpp-bindings-build/src/link_kind.rs @@ -0,0 +1,7 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LinkKind { + Unspecified, + Static, + Dylib, + Framework, +} diff --git a/llama-cpp-bindings-build/src/native_build.rs b/llama-cpp-bindings-build/src/native_build.rs new file mode 100644 index 000000000..650ac725d --- /dev/null +++ b/llama-cpp-bindings-build/src/native_build.rs @@ -0,0 +1,317 @@ +use std::io::Write; + +use crate::bindings_generation::BindingsGeneration; +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::cargo_directive::CargoDirective; +use crate::cmake_build::CmakeBuild; +use crate::cmake_configuration::CmakeConfiguration; +use crate::cmake_file_api::CmakeFileApi; +use crate::emit_directives::emit_directives; +use crate::link_directives::link_directives; +use crate::openmp_linking::OpenmpLinking; +use crate::rebuild_triggers::rebuild_triggers; +use crate::wrapper_compilation::WrapperCompilation; + +/// Every step of the native build, decided before any of them runs. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NativeBuild { + pub bindings: BindingsGeneration, + pub wrappers: WrapperCompilation, + pub cmake: CmakeBuild, + pub directives: Vec, + pub openmp: OpenmpLinking, + pub include_debug_messages: bool, +} + +impl NativeBuild { + /// # Errors + /// + /// Returns the [`BuildError`] of [`CmakeConfiguration::of`] or [`link_directives`]. + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Result { + let CmakeConfiguration { + defines, + required_directories, + directives: cmake_directives, + } = CmakeConfiguration::of(context, environment)?; + + Ok(Self { + bindings: BindingsGeneration::of(context, environment), + wrappers: WrapperCompilation::of(context, environment), + cmake: CmakeBuild { + source_dir: context.llama_src(), + out_dir: context.cmake_dir(), + file_api: CmakeFileApi { + build_dir: context.cmake_build_dir(), + }, + target_triple: context.target_triple.clone(), + host_triple: context.host_triple.clone(), + defines, + required_directories, + }, + directives: [ + context.debug_messages(), + rebuild_triggers(context), + cmake_directives, + link_directives(context, environment)?, + ] + .concat(), + openmp: OpenmpLinking::of(context, environment), + include_debug_messages: context.include_debug_messages, + }) + } + + /// Generates the bindings, compiles the wrappers, builds llama.cpp, and only then writes the + /// cargo directives to `output`, with the OpenMP runtime `CMake` resolved linked after every + /// planned library. + /// + /// # Errors + /// + /// Returns the [`BuildError`] of the first step that fails. + pub fn run(self, output: &mut impl Write) -> Result<(), BuildError> { + let bindings_directives = self.bindings.generate()?; + + self.wrappers.compile()?; + + let cmake_cache = self.cmake.build()?; + let openmp_directives = self.openmp.directives(&cmake_cache)?; + + emit_directives( + &[self.directives, openmp_directives, bindings_directives].concat(), + self.include_debug_messages, + output, + ) + } +} + +#[cfg(test)] +mod tests { + use std::env; + use std::fs; + use std::path::Path; + use std::path::PathBuf; + use std::process; + + use super::NativeBuild; + use crate::bindings_generation::BindingsGeneration; + use crate::cargo_directive::CargoDirective; + use crate::cmake_build::CmakeBuild; + use crate::cmake_define::CmakeDefine; + use crate::cmake_file_api::CmakeFileApi; + use crate::link_kind::LinkKind; + use crate::native_library::NativeLibrary; + use crate::openmp_linking::OpenmpLinking; + use crate::openmp_runtime_source::OpenmpRuntimeSource; + use crate::target_os::TargetOs; + use crate::wrapper_compilation::WrapperCompilation; + + #[cfg(all(target_arch = "x86_64", target_os = "linux", target_env = "gnu"))] + const HOST_TRIPLE: &str = "x86_64-unknown-linux-gnu"; + #[cfg(all(target_arch = "aarch64", target_os = "linux", target_env = "gnu"))] + const HOST_TRIPLE: &str = "aarch64-unknown-linux-gnu"; + #[cfg(all(target_arch = "x86_64", target_os = "macos"))] + const HOST_TRIPLE: &str = "x86_64-apple-darwin"; + #[cfg(all(target_arch = "aarch64", target_os = "macos"))] + const HOST_TRIPLE: &str = "aarch64-apple-darwin"; + + const FIXTURES: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures"); + + const MANIFEST: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml"); + + fn fixture_build(root: &Path) -> NativeBuild { + NativeBuild { + bindings: BindingsGeneration { + header: PathBuf::from(FIXTURES).join("bindings/wrapper.h"), + clang_args: Vec::new(), + output: root.join("bindings.rs"), + }, + wrappers: WrapperCompilation { + sources: vec![PathBuf::from(FIXTURES).join("wrapper/wrapper_fixture.cpp")], + include_dirs: vec![PathBuf::from(FIXTURES).join("wrapper")], + flags: vec![format!("-isystem{FIXTURES}/wrapper/vendor")], + cpp_standard: "c++17", + defines: vec!["WRAPPER_FIXTURE_DEFINE"], + cpp_stdlib_linked_separately: false, + target_triple: HOST_TRIPLE.to_owned(), + host_triple: HOST_TRIPLE.to_owned(), + opt_level: "0".to_owned(), + debug_info: "false".to_owned(), + out_dir: root.join("wrapper"), + }, + cmake: CmakeBuild { + source_dir: PathBuf::from(FIXTURES).join("cmake_project"), + out_dir: root.join("cmake"), + file_api: CmakeFileApi { + build_dir: root.join("cmake/build"), + }, + target_triple: HOST_TRIPLE.to_owned(), + host_triple: HOST_TRIPLE.to_owned(), + defines: vec![CmakeDefine::new("FIXTURE_LIBRARY_NAME", "fixture")], + required_directories: vec![root.join("cmake/backends")], + }, + directives: vec![ + CargoDirective::LinkSearchNative(root.join("cmake/lib")), + CargoDirective::LinkLibrary(NativeLibrary { + link_kind: LinkKind::Static, + name: "fixture", + }), + CargoDirective::Debug("fixture planned".to_owned()), + ], + openmp: OpenmpLinking { + source: OpenmpRuntimeSource::BundledGgml, + target_os: TargetOs::Linux, + }, + include_debug_messages: false, + } + } + + fn successful_directives(root: &Path) -> String { + format!( + "cargo:rustc-link-search=native={root}/cmake/lib\n\ + cargo:rustc-link-lib=static=fixture\n\ + cargo:rustc-link-search=native={root}/cmake/lib\n\ + cargo:rustc-link-lib=dylib:+verbatim=libfixture.a\n\ + cargo:rerun-if-env-changed=TARGET\n\ + cargo:rerun-if-env-changed=BINDGEN_EXTRA_CLANG_ARGS\n\ + cargo:rerun-if-changed={FIXTURES}/bindings/wrapper.h\n\ + cargo:rerun-if-changed={FIXTURES}/bindings/llama_types.h\n", + root = root.display() + ) + } + + fn build_root(name: &str) -> PathBuf { + let root = + env::temp_dir().join(format!("llama-cpp-bindings-build-{name}-{}", process::id())); + + fs::create_dir_all(&root).unwrap(); + + root + } + + #[test] + fn a_successful_build_writes_every_artifact_whoever_links_the_cxx_runtime_then_the_directives() + { + for cpp_stdlib_linked_separately in [false, true] { + let root = build_root(&format!("successful-{cpp_stdlib_linked_separately}")); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + native_build.wrappers.cpp_stdlib_linked_separately = cpp_stdlib_linked_separately; + native_build.run(&mut output).unwrap(); + + assert_eq!( + String::from_utf8(output).unwrap(), + successful_directives(&root) + ); + assert_eq!( + [ + root.join("bindings.rs").is_file(), + root.join("wrapper/libllama_cpp_bindings_sys_common_wrapper.a") + .is_file(), + root.join("cmake/lib/libfixture.a").is_file(), + root.join("cmake/backends").is_dir(), + ], + [true; 4] + ); + fs::remove_dir_all(root).unwrap(); + } + } + + #[test] + fn a_build_tree_left_by_a_relocated_source_still_reports_the_cmake_cache() { + let root = build_root("relocated-source"); + let relocated_source = root.join("relocated_cmake_project"); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + fixture_build(&root).run(&mut Vec::new()).unwrap(); + fs::create_dir_all(&relocated_source).unwrap(); + for file_name in ["CMakeLists.txt", "fixture.c"] { + fs::copy( + PathBuf::from(FIXTURES) + .join("cmake_project") + .join(file_name), + relocated_source.join(file_name), + ) + .unwrap(); + } + native_build.cmake.source_dir = relocated_source; + native_build.run(&mut output).unwrap(); + + assert_eq!( + String::from_utf8(output).unwrap(), + successful_directives(&root) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn a_failed_wrapper_compilation_stops_the_build_before_cmake_and_any_directive() { + let root = build_root("failed-wrappers"); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + native_build.wrappers.out_dir = PathBuf::from(MANIFEST); + + assert_eq!( + native_build + .run(&mut output) + .err() + .map(|error| error.to_string()), + Some( + "native wrapper compilation failed: IOError: File exists (os error 17)".to_owned() + ) + ); + assert_eq!( + [output.is_empty(), root.join("cmake").exists()], + [true, false] + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn a_cmake_build_that_cannot_create_its_directories_emits_no_directive() { + let root = build_root("failed-cmake"); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + native_build.cmake.required_directories = vec![PathBuf::from(MANIFEST).join("backends")]; + + assert_eq!( + native_build + .run(&mut output) + .err() + .map(|error| error.to_string()), + Some(format!( + "filesystem operation failed for {MANIFEST}/backends: Not a directory (os error \ + 20)" + )) + ); + assert_eq!(output, Vec::::new()); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn an_apple_build_whose_ggml_needs_an_openmp_runtime_emits_no_directive() { + let root = build_root("apple-openmp"); + let mut native_build = fixture_build(&root); + let mut output = Vec::new(); + + native_build.openmp.target_os = TargetOs::Apple; + + assert_eq!( + native_build + .run(&mut output) + .err() + .map(|error| error.to_string()), + Some(format!( + "ggml needs the OpenMP runtime [{:?}], which Apple's linker cannot link by its \ + exact file name", + root.join("cmake/lib/libfixture.a") + )) + ); + assert_eq!(output, Vec::::new()); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/llama-cpp-bindings-build/src/native_library.rs b/llama-cpp-bindings-build/src/native_library.rs index 71ed20e29..298569942 100644 --- a/llama-cpp-bindings-build/src/native_library.rs +++ b/llama-cpp-bindings-build/src/native_library.rs @@ -1,5 +1,48 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +use crate::link_kind::LinkKind; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct NativeLibrary { - pub link_kind: &'static str, + pub link_kind: LinkKind, pub name: &'static str, } + +impl Display for NativeLibrary { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + let Self { link_kind, name } = self; + + match link_kind { + LinkKind::Unspecified => write!(formatter, "{name}"), + LinkKind::Static => write!(formatter, "static={name}"), + LinkKind::Dylib => write!(formatter, "dylib={name}"), + LinkKind::Framework => write!(formatter, "framework={name}"), + } + } +} + +#[cfg(test)] +mod tests { + use super::NativeLibrary; + use crate::link_kind::LinkKind; + + #[test] + fn every_link_kind_prefixes_the_library_the_way_cargo_expects() { + assert_eq!( + [ + LinkKind::Unspecified, + LinkKind::Static, + LinkKind::Dylib, + LinkKind::Framework + ] + .map(|link_kind| NativeLibrary { + link_kind, + name: "llama" + } + .to_string()), + ["llama", "static=llama", "dylib=llama", "framework=llama"] + ); + } +} diff --git a/llama-cpp-bindings-build/src/native_profile.rs b/llama-cpp-bindings-build/src/native_profile.rs new file mode 100644 index 000000000..2186083d7 --- /dev/null +++ b/llama-cpp-bindings-build/src/native_profile.rs @@ -0,0 +1 @@ +pub const NATIVE_PROFILE: &str = "Release"; diff --git a/llama-cpp-bindings-build/src/openmp_linking.rs b/llama-cpp-bindings-build/src/openmp_linking.rs new file mode 100644 index 000000000..6a2d0c10f --- /dev/null +++ b/llama-cpp-bindings-build/src/openmp_linking.rs @@ -0,0 +1,122 @@ +use std::path::PathBuf; + +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::cargo_directive::CargoDirective; +use crate::cmake_cache::CmakeCache; +use crate::openmp_runtime_source::OpenmpRuntimeSource; +use crate::target_os::TargetOs; + +fn link_by_file_name(library: PathBuf) -> Result<[CargoDirective; 2], BuildError> { + match (library.parent(), library.file_name()) { + (Some(directory), Some(file_name)) => Ok([ + CargoDirective::LinkSearchNative(directory.to_path_buf()), + CargoDirective::LinkDylibByFileName(PathBuf::from(file_name)), + ]), + _ => Err(BuildError::OpenmpRuntimeLibraryWithoutFileName { library }), + } +} + +/// Links the OpenMP runtime `CMake` resolved for ggml, which is only known once `CMake` has +/// configured the build. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OpenmpLinking { + pub source: OpenmpRuntimeSource, + pub target_os: TargetOs, +} + +impl OpenmpLinking { + #[must_use] + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Self { + Self { + source: OpenmpRuntimeSource::of(context, environment), + target_os: context.target_os.clone(), + } + } + + /// Links each runtime library by its exact file name from its own directory, after every + /// library that needs it. + /// + /// # Errors + /// + /// Returns the [`BuildError`] of [`OpenmpRuntimeSource::libraries`], + /// [`BuildError::OpenmpRuntimeUnlinkableOnApple`] when an Apple target needs a runtime, or + /// [`BuildError::OpenmpRuntimeLibraryWithoutFileName`] when a runtime library path names no + /// file. + pub fn directives(&self, cache: &CmakeCache) -> Result, BuildError> { + let libraries = self.source.libraries(cache)?; + + if !libraries.is_empty() && matches!(self.target_os, TargetOs::Apple) { + return Err(BuildError::OpenmpRuntimeUnlinkableOnApple { libraries }); + } + + libraries + .into_iter() + .map(link_by_file_name) + .collect::, BuildError>>() + .map(|directives| directives.into_iter().flatten().collect()) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::OpenmpLinking; + use crate::cargo_directive::CargoDirective; + use crate::cmake_cache::CmakeCache; + use crate::cmake_cache::tests::entry; + use crate::cmake_cache_entry::CmakeCacheEntry; + use crate::openmp_runtime_source::OpenmpRuntimeSource; + use crate::target_os::TargetOs; + + fn linux_directives(entries: Vec) -> Result, String> { + OpenmpLinking { + source: OpenmpRuntimeSource::BundledGgml, + target_os: TargetOs::Linux, + } + .directives(&CmakeCache { entries }) + .map_err(|error| error.to_string()) + } + + #[test] + fn each_runtime_library_is_linked_by_its_exact_file_name_from_its_directory() { + assert_eq!( + linux_directives(vec![ + entry("GGML_OPENMP_ENABLED", "ON"), + entry("OpenMP_C_LIB_NAMES", "gomp;pthread"), + entry("OpenMP_CXX_LIB_NAMES", "gomp;pthread"), + entry("OpenMP_gomp_LIBRARY", "/usr/lib/gcc/libgomp.so"), + entry("OpenMP_pthread_LIBRARY", "/usr/lib/libpthread.a"), + ]), + Ok(vec![ + CargoDirective::LinkSearchNative(PathBuf::from("/usr/lib/gcc")), + CargoDirective::LinkDylibByFileName(PathBuf::from("libgomp.so")), + CargoDirective::LinkSearchNative(PathBuf::from("/usr/lib")), + CargoDirective::LinkDylibByFileName(PathBuf::from("libpthread.a")), + ]) + ); + } + + #[test] + fn a_cache_without_the_bundled_ggml_openmp_switch_is_refused() { + assert_eq!( + linux_directives(Vec::new()), + Err("the CMake cache has no entry GGML_OPENMP_ENABLED".to_owned()) + ); + } + + #[test] + fn a_runtime_library_path_without_a_file_name_is_refused() { + assert_eq!( + linux_directives(vec![ + entry("GGML_OPENMP_ENABLED", "ON"), + entry("OpenMP_C_LIB_NAMES", "gomp"), + entry("OpenMP_CXX_LIB_NAMES", "gomp"), + entry("OpenMP_gomp_LIBRARY", "/"), + ]), + Err("the OpenMP runtime library / names no file".to_owned()) + ); + } +} diff --git a/llama-cpp-bindings-build/src/openmp_runtime_source.rs b/llama-cpp-bindings-build/src/openmp_runtime_source.rs new file mode 100644 index 000000000..5a808fadc --- /dev/null +++ b/llama-cpp-bindings-build/src/openmp_runtime_source.rs @@ -0,0 +1,208 @@ +use std::path::PathBuf; + +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::cmake_cache::CmakeCache; + +const BUNDLED_GGML_OPENMP_SWITCH: &str = "GGML_OPENMP_ENABLED"; + +const C_RUNTIME_LIBRARY_NAMES: &str = "OpenMP_C_LIB_NAMES"; + +const CXX_RUNTIME_LIBRARY_NAMES: &str = "OpenMP_CXX_LIB_NAMES"; + +/// Where the build learns which OpenMP runtime libraries a statically linked ggml needs. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OpenmpRuntimeSource { + SharedLibraries, + BundledGgml, + SystemGgml, +} + +impl OpenmpRuntimeSource { + #[must_use] + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Self { + match ( + context.build_shared_libs, + environment.has(BuildFeature::SystemGgml), + environment.has(BuildFeature::SystemGgmlStatic), + ) { + (true, _, _) | (false, true, false) => Self::SharedLibraries, + (false, true, true) => Self::SystemGgml, + (false, false, _) => Self::BundledGgml, + } + } + + fn needs_runtime(self, cache: &CmakeCache) -> Result { + match self { + Self::SharedLibraries => Ok(false), + Self::BundledGgml => cache.switch(BUNDLED_GGML_OPENMP_SWITCH), + Self::SystemGgml => Ok(cache.contains(C_RUNTIME_LIBRARY_NAMES)), + } + } + + /// The OpenMP runtime libraries `CMake` resolved for ggml, which only a statically linked + /// ggml leaves for the bindings to link. + /// + /// # Errors + /// + /// Returns the [`BuildError`] of [`CmakeCache::switch`], [`CmakeCache::list`], or + /// [`CmakeCache::value`] when the cache does not tell whether or where ggml's OpenMP + /// runtime is. + pub fn libraries(self, cache: &CmakeCache) -> Result, BuildError> { + if !self.needs_runtime(cache)? { + return Ok(Vec::new()); + } + + let mut names = Vec::new(); + + for library_names in [C_RUNTIME_LIBRARY_NAMES, CXX_RUNTIME_LIBRARY_NAMES] { + for name in cache.list(library_names)? { + if !names.contains(&name) { + names.push(name); + } + } + } + + names + .into_iter() + .map(|name| { + cache + .value(&format!("OpenMP_{name}_LIBRARY")) + .map(PathBuf::from) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use enumflags2::BitFlags; + + use super::OpenmpRuntimeSource; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_feature::BuildFeature; + use crate::cmake_cache::CmakeCache; + use crate::cmake_cache::tests::entry; + use crate::cmake_cache_entry::CmakeCacheEntry; + + fn gnu_runtime() -> Vec { + vec![ + entry("OpenMP_C_LIB_NAMES", "gomp;pthread"), + entry("OpenMP_CXX_LIB_NAMES", "gomp"), + entry("OpenMP_gomp_LIBRARY", "/usr/lib/gcc/libgomp.so"), + entry("OpenMP_pthread_LIBRARY", "/usr/lib/libpthread.a"), + ] + } + + fn bundled_ggml_with_openmp(switch: &str) -> CmakeCache { + CmakeCache { + entries: [vec![entry("GGML_OPENMP_ENABLED", switch)], gnu_runtime()].concat(), + } + } + + fn gnu_runtime_libraries() -> Vec { + vec![ + PathBuf::from("/usr/lib/gcc/libgomp.so"), + PathBuf::from("/usr/lib/libpthread.a"), + ] + } + + fn bundled_ggml_failure(entries: Vec) -> Option { + OpenmpRuntimeSource::BundledGgml + .libraries(&CmakeCache { entries }) + .err() + .map(|error| error.to_string()) + } + + #[test] + fn only_a_statically_linked_ggml_leaves_its_runtime_to_the_bindings() { + assert_eq!( + [ + BitFlags::from(BuildFeature::DynamicLink), + BitFlags::from(BuildFeature::SystemGgml), + BuildFeature::SystemGgml | BuildFeature::SystemGgmlStatic, + BitFlags::empty(), + ] + .map(|features| { + let environment = cargo_environment(features, &[]); + + OpenmpRuntimeSource::of(&context_of(&environment), &environment) + }), + [ + OpenmpRuntimeSource::SharedLibraries, + OpenmpRuntimeSource::SharedLibraries, + OpenmpRuntimeSource::SystemGgml, + OpenmpRuntimeSource::BundledGgml, + ] + ); + } + + #[test] + fn shared_libraries_need_no_runtime_from_the_cache() { + assert_eq!( + OpenmpRuntimeSource::SharedLibraries + .libraries(&CmakeCache { + entries: Vec::new() + }) + .unwrap(), + Vec::::new() + ); + } + + #[test] + fn a_bundled_ggml_with_openmp_needs_every_runtime_library_of_both_languages_once() { + assert_eq!( + OpenmpRuntimeSource::BundledGgml + .libraries(&bundled_ggml_with_openmp("ON")) + .unwrap(), + gnu_runtime_libraries() + ); + } + + #[test] + fn a_bundled_ggml_without_openmp_needs_no_runtime() { + assert_eq!( + OpenmpRuntimeSource::BundledGgml + .libraries(&bundled_ggml_with_openmp("OFF")) + .unwrap(), + Vec::::new() + ); + } + + #[test] + fn a_system_ggml_needs_the_runtime_only_when_its_config_searched_for_openmp() { + assert_eq!( + [Vec::new(), gnu_runtime()].map(|entries| { + OpenmpRuntimeSource::SystemGgml + .libraries(&CmakeCache { entries }) + .unwrap() + }), + [Vec::new(), gnu_runtime_libraries()] + ); + } + + #[test] + fn a_bundled_ggml_with_openmp_but_no_runtime_library_list_is_refused() { + assert_eq!( + bundled_ggml_failure(vec![entry("GGML_OPENMP_ENABLED", "ON")]), + Some("the CMake cache has no entry OpenMP_C_LIB_NAMES".to_owned()) + ); + } + + #[test] + fn a_runtime_library_cmake_did_not_locate_is_refused() { + assert_eq!( + bundled_ggml_failure(vec![ + entry("GGML_OPENMP_ENABLED", "ON"), + entry("OpenMP_C_LIB_NAMES", "gomp"), + entry("OpenMP_CXX_LIB_NAMES", "gomp"), + ]), + Some("the CMake cache has no entry OpenMP_gomp_LIBRARY".to_owned()) + ); + } +} diff --git a/llama-cpp-bindings-build/src/privatized_field.rs b/llama-cpp-bindings-build/src/privatized_field.rs new file mode 100644 index 000000000..d1a637b1e --- /dev/null +++ b/llama-cpp-bindings-build/src/privatized_field.rs @@ -0,0 +1,20 @@ +/// A field of a C struct that the generated bindings keep private, so callers go through the +/// typed API instead. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PrivatizedField { + pub type_name: &'static str, + pub field_name: &'static str, +} + +impl PrivatizedField { + pub const ALL: &'static [Self] = &[ + Self { + type_name: "llama_context_params", + field_name: "defrag_thold", + }, + Self { + type_name: "mtmd_context_params", + field_name: "image_marker", + }, + ]; +} diff --git a/llama-cpp-bindings-build/src/rebuild_tracking.rs b/llama-cpp-bindings-build/src/rebuild_tracking.rs deleted file mode 100644 index cf5523592..000000000 --- a/llama-cpp-bindings-build/src/rebuild_tracking.rs +++ /dev/null @@ -1,15 +0,0 @@ -use std::path::Path; - -use llama_cpp_wrapper_sources::wrapper_headers::WRAPPER_HEADERS; -use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; - -pub fn register_rebuild_triggers(llama_src: &Path) { - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-env-changed=CUDAARCHS"); - - for path in WRAPPER_HEADERS.iter().chain(WRAPPER_SOURCES) { - println!("cargo:rerun-if-changed={path}"); - } - - println!("cargo:rerun-if-changed={}", llama_src.display()); -} diff --git a/llama-cpp-bindings-build/src/rebuild_triggers.rs b/llama-cpp-bindings-build/src/rebuild_triggers.rs new file mode 100644 index 000000000..1617ed21f --- /dev/null +++ b/llama-cpp-bindings-build/src/rebuild_triggers.rs @@ -0,0 +1,66 @@ +use llama_cpp_wrapper_sources::wrapper_headers::WRAPPER_HEADERS; +use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + +use crate::build_context::BuildContext; +use crate::cargo_directive::CargoDirective; + +/// The files and variables whose change makes cargo run the native build again. +#[must_use] +pub fn rebuild_triggers(context: &BuildContext) -> Vec { + [ + CargoDirective::RerunIfChanged(context.sys_dir.join("build.rs")), + CargoDirective::RerunIfEnvChanged("CUDAARCHS".to_owned()), + ] + .into_iter() + .chain( + WRAPPER_HEADERS + .iter() + .chain(WRAPPER_SOURCES) + .map(|wrapper_file| CargoDirective::RerunIfChanged(context.sys_dir.join(wrapper_file))), + ) + .chain([CargoDirective::RerunIfChanged(context.llama_src())]) + .collect() +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use enumflags2::BitFlags; + + use llama_cpp_wrapper_sources::wrapper_headers::WRAPPER_HEADERS; + use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + + use super::rebuild_triggers; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::cargo_directive::CargoDirective; + + #[test] + fn the_build_script_cuda_architectures_every_wrapper_file_and_llama_cpp_trigger_a_rebuild() { + let triggers = rebuild_triggers(&context_of(&cargo_environment(BitFlags::empty(), &[]))); + + assert_eq!( + triggers, + [ + vec![ + CargoDirective::RerunIfChanged(PathBuf::from("/sys/build.rs")), + CargoDirective::RerunIfEnvChanged("CUDAARCHS".to_owned()), + ], + WRAPPER_HEADERS + .iter() + .chain(WRAPPER_SOURCES) + .map(|wrapper_file| { + CargoDirective::RerunIfChanged(PathBuf::from(format!( + "/sys/{wrapper_file}" + ))) + }) + .collect(), + vec![CargoDirective::RerunIfChanged(PathBuf::from( + "/sys/llama.cpp" + ))], + ] + .concat() + ); + } +} diff --git a/llama-cpp-bindings-build/src/target_os.rs b/llama-cpp-bindings-build/src/target_os.rs index 240a3de8d..c346f9b37 100644 --- a/llama-cpp-bindings-build/src/target_os.rs +++ b/llama-cpp-bindings-build/src/target_os.rs @@ -1,67 +1,139 @@ -use crate::apple_variant::AppleVariant; +use crate::android_ndk::AndroidNdk; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::host_os::HostOs; -#[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum TargetOs { - Apple(AppleVariant), + Apple, Linux, - Android, + Android(AndroidNdk), } impl TargetOs { - #[must_use] - pub fn from_cargo_cfg(cargo_cfg_target_os: &str) -> Option { - match cargo_cfg_target_os { - "macos" => Some(Self::Apple(AppleVariant::MacOS)), - "ios" | "tvos" | "watchos" | "visionos" => Some(Self::Apple(AppleVariant::Other)), - "android" => Some(Self::Android), - "linux" => Some(Self::Linux), - _ => None, - } - } + /// The operating system cargo builds for, with the NDK it needs when that is Android. + /// + /// # Errors + /// + /// Returns [`BuildError::Environment`] when `CARGO_CFG_TARGET_OS` cannot be read, + /// [`BuildError::UnsupportedTargetOs`] for an operating system this crate does not build + /// for, or the [`BuildError`] of [`AndroidNdk::detect`] when an Android target has no + /// usable NDK. + pub fn detect( + environment: &BuildEnvironment, + target_triple: &str, + cargo_cfg_target_arch: &str, + host_os: HostOs, + ) -> Result { + let cargo_cfg_target_os = environment.required("CARGO_CFG_TARGET_OS")?; - #[must_use] - pub const fn is_android(self) -> bool { - matches!(self, Self::Android) + match cargo_cfg_target_os.as_str() { + "macos" | "ios" | "tvos" | "watchos" | "visionos" => Ok(Self::Apple), + "android" => Ok(Self::Android(AndroidNdk::detect( + environment, + target_triple, + cargo_cfg_target_arch, + host_os, + )?)), + "linux" => Ok(Self::Linux), + _ => Err(BuildError::UnsupportedTargetOs { + cargo_cfg_target_os, + }), + } } } #[cfg(test)] mod tests { + use std::ffi::OsString; + + use enumflags2::BitFlags; + use super::TargetOs; - use crate::apple_variant::AppleVariant; + use crate::android_ndk::AndroidNdk; + use crate::android_ndk::tests::FIXTURE_ANDROID_NDK; + use crate::build_environment::tests::environment_of; + use crate::build_environment::tests::variable; + use crate::host_os::HostOs; + + fn detected(cargo_cfg_target_os: &str) -> Result { + let environment = environment_of( + BitFlags::empty(), + &[ + variable("CARGO_CFG_TARGET_OS", OsString::from(cargo_cfg_target_os)), + variable("ANDROID_NDK_HOME", OsString::from(FIXTURE_ANDROID_NDK)), + ], + ); + + TargetOs::detect( + &environment, + "aarch64-linux-android", + "aarch64", + HostOs::Linux, + ) + .map_err(|error| error.to_string()) + } #[test] - fn macos_is_distinguished_from_the_other_apple_platforms() { + fn every_apple_platform_is_an_apple_target() { assert_eq!( - TargetOs::from_cargo_cfg("macos"), - Some(TargetOs::Apple(AppleVariant::MacOS)) + ["macos", "ios", "tvos", "watchos", "visionos"].map(detected), + [(); 5].map(|()| Ok(TargetOs::Apple)) ); - - for apple_os in ["ios", "tvos", "watchos", "visionos"] { - assert_eq!( - TargetOs::from_cargo_cfg(apple_os), - Some(TargetOs::Apple(AppleVariant::Other)), - "{apple_os} must classify as a non-macOS Apple target" - ); - } } #[test] - fn android_is_not_mistaken_for_linux() { - assert_eq!(TargetOs::from_cargo_cfg("android"), Some(TargetOs::Android)); - assert_eq!(TargetOs::from_cargo_cfg("linux"), Some(TargetOs::Linux)); + fn android_is_not_mistaken_for_linux_and_carries_its_ndk() { + let environment = environment_of( + BitFlags::empty(), + &[variable( + "ANDROID_NDK_HOME", + OsString::from(FIXTURE_ANDROID_NDK), + )], + ); + + assert_eq!( + [detected("android"), detected("linux")], + [ + Ok(TargetOs::Android( + AndroidNdk::detect( + &environment, + "aarch64-linux-android", + "aarch64", + HostOs::Linux + ) + .unwrap() + )), + Ok(TargetOs::Linux) + ] + ); } #[test] - fn only_android_needs_the_android_stdlib_handling() { - assert!(TargetOs::Android.is_android()); - assert!(!TargetOs::Linux.is_android()); + fn an_unsupported_target_os_is_refused() { + assert_eq!( + detected("freebsd"), + Err("unsupported target operating system: freebsd".to_owned()) + ); } #[test] - fn an_unsupported_target_os_is_rejected() { - for unsupported_target_os in ["freebsd", "windows"] { - assert_eq!(TargetOs::from_cargo_cfg(unsupported_target_os), None); - } + fn an_android_target_without_an_ndk_is_refused() { + assert_eq!( + TargetOs::detect( + &environment_of( + BitFlags::empty(), + &[variable("CARGO_CFG_TARGET_OS", "android")] + ), + "aarch64-linux-android", + "aarch64", + HostOs::Linux + ) + .map_err(|error| error.to_string()), + Err( + "Android NDK not found for target aarch64-linux-android. Set ANDROID_NDK_HOME." + .to_owned() + ) + ); } } diff --git a/llama-cpp-bindings-build/src/wrapper_compilation.rs b/llama-cpp-bindings-build/src/wrapper_compilation.rs new file mode 100644 index 000000000..657474f99 --- /dev/null +++ b/llama-cpp-bindings-build/src/wrapper_compilation.rs @@ -0,0 +1,173 @@ +use std::path::PathBuf; + +use llama_cpp_wrapper_sources::cpp_standard::CPP_STANDARD; +use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; +use llama_cpp_wrapper_sources::system_include_flags::system_include_flags; +use llama_cpp_wrapper_sources::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; +use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + +use crate::build_context::BuildContext; +use crate::build_environment::BuildEnvironment; +use crate::build_error::BuildError; +use crate::build_feature::BuildFeature; +use crate::target_os::TargetOs; + +const WRAPPER_LIBRARY: &str = "llama_cpp_bindings_sys_common_wrapper"; + +/// Compiles the C++ wrapper sources into the static wrapper library in `out_dir`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct WrapperCompilation { + pub sources: Vec, + pub include_dirs: Vec, + pub flags: Vec, + pub cpp_standard: &'static str, + pub defines: Vec<&'static str>, + pub cpp_stdlib_linked_separately: bool, + pub target_triple: String, + pub host_triple: String, + pub opt_level: String, + pub debug_info: String, + pub out_dir: PathBuf, +} + +impl WrapperCompilation { + #[must_use] + pub fn of(context: &BuildContext, environment: &BuildEnvironment) -> Self { + Self { + sources: WRAPPER_SOURCES + .iter() + .map(|source| context.sys_dir.join(source)) + .collect(), + include_dirs: WRAPPER_INCLUDE_DIRS + .iter() + .map(|include_dir| context.sys_dir.join(include_dir)) + .collect(), + flags: system_include_flags(&context.sys_dir), + cpp_standard: CPP_STANDARD, + defines: if environment.has(BuildFeature::FaultInjection) { + vec![FAULT_INJECTION_DEFINE] + } else { + Vec::new() + }, + cpp_stdlib_linked_separately: matches!(context.target_os, TargetOs::Android(_)) + && environment.has(BuildFeature::StaticStdcxx), + target_triple: context.target_triple.clone(), + host_triple: context.host_triple.clone(), + opt_level: context.opt_level.clone(), + debug_info: context.debug_info.clone(), + out_dir: context.out_dir.clone(), + } + } + + /// # Errors + /// + /// Returns [`BuildError::NativeWrapper`] when the wrapper library cannot be compiled. + pub fn compile(&self) -> Result<(), BuildError> { + let mut build = cc::Build::new(); + + build.cpp(true).pic(true).includes(&self.include_dirs); + + for flag in &self.flags { + build.flag(flag); + } + + build.std(self.cpp_standard); + + for define in &self.defines { + build.define(define, None); + } + + build.files(&self.sources); + + if self.cpp_stdlib_linked_separately { + build.cpp_link_stdlib(None); + } + + build + .target(&self.target_triple) + .host(&self.host_triple) + .opt_level_str(&self.opt_level) + .debug_str(&self.debug_info) + .out_dir(&self.out_dir) + .try_compile(WRAPPER_LIBRARY) + .map_err(BuildError::NativeWrapper) + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::path::PathBuf; + + use enumflags2::BitFlags; + + use llama_cpp_wrapper_sources::fault_injection_define::FAULT_INJECTION_DEFINE; + use llama_cpp_wrapper_sources::system_include_flags::system_include_flags; + use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + + use super::WrapperCompilation; + use crate::build_context::tests::android_overrides; + use crate::build_context::tests::cargo_environment; + use crate::build_context::tests::context_of; + use crate::build_environment::tests::EnvironmentVariable; + use crate::build_feature::BuildFeature; + + fn planned( + features: BitFlags, + overrides: &[EnvironmentVariable], + ) -> WrapperCompilation { + let environment = cargo_environment(features, overrides); + + WrapperCompilation::of(&context_of(&environment), &environment) + } + + fn linux_compilation() -> WrapperCompilation { + WrapperCompilation { + sources: WRAPPER_SOURCES + .iter() + .map(|source| PathBuf::from(format!("/sys/{source}"))) + .collect(), + include_dirs: vec![PathBuf::from("/sys/.")], + flags: system_include_flags(Path::new("/sys")), + cpp_standard: "c++17", + defines: Vec::new(), + cpp_stdlib_linked_separately: false, + target_triple: "x86_64-unknown-linux-gnu".to_owned(), + host_triple: "x86_64-unknown-linux-gnu".to_owned(), + opt_level: "3".to_owned(), + debug_info: "false".to_owned(), + out_dir: PathBuf::from("/out"), + } + } + + #[test] + fn the_wrappers_compile_as_cxx17_with_the_vendored_headers_as_system_includes() { + assert_eq!(planned(BitFlags::empty(), &[]), linux_compilation()); + } + + #[test] + fn fault_injection_defines_its_switch() { + assert_eq!( + planned(BitFlags::from(BuildFeature::FaultInjection), &[]), + WrapperCompilation { + defines: vec![FAULT_INJECTION_DEFINE], + ..linux_compilation() + } + ); + } + + #[test] + fn a_static_android_stl_is_linked_separately() { + assert_eq!( + planned( + BitFlags::from(BuildFeature::StaticStdcxx), + &android_overrides() + ), + WrapperCompilation { + cpp_stdlib_linked_separately: true, + target_triple: "aarch64-linux-android".to_owned(), + ..linux_compilation() + } + ); + } +} diff --git a/llama-cpp-bindings-build/src/x86_ggml_switch.rs b/llama-cpp-bindings-build/src/x86_ggml_switch.rs new file mode 100644 index 000000000..6398e1299 --- /dev/null +++ b/llama-cpp-bindings-build/src/x86_ggml_switch.rs @@ -0,0 +1,46 @@ +use crate::cmake_define::CmakeDefine; + +/// A ggml x86 instruction-set switch and the rustc target features that must all be enabled +/// to turn it on. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct X86GgmlSwitch { + pub name: &'static str, + pub target_features: &'static [&'static str], +} + +impl X86GgmlSwitch { + #[must_use] + pub fn define_for(&self, enabled_features: &[&str]) -> CmakeDefine { + CmakeDefine::switch( + self.name, + self.target_features + .iter() + .all(|target_feature| enabled_features.contains(target_feature)), + ) + } +} + +#[cfg(test)] +mod tests { + use super::X86GgmlSwitch; + use crate::cmake_define::CmakeDefine; + + const AVX512: X86GgmlSwitch = X86GgmlSwitch { + name: "GGML_AVX512", + target_features: &["avx512f", "avx512bw"], + }; + + #[test] + fn a_switch_is_on_only_when_every_target_feature_it_needs_is_enabled() { + assert_eq!( + [ + AVX512.define_for(&["avx512f", "avx512bw", "avx2"]), + AVX512.define_for(&["avx512f", "avx2"]), + ], + [ + CmakeDefine::switch("GGML_AVX512", true), + CmakeDefine::switch("GGML_AVX512", false), + ] + ); + } +} diff --git a/llama-cpp-bindings-sys/Cargo.toml b/llama-cpp-bindings-sys/Cargo.toml index b4ce0791e..4c2b20e94 100644 --- a/llama-cpp-bindings-sys/Cargo.toml +++ b/llama-cpp-bindings-sys/Cargo.toml @@ -56,14 +56,15 @@ include = [ "/llama.cpp/vendor", "/llama.cpp/CMakeLists.txt", + "/llama.cpp/LICENSE", "/llama.cpp/common/CMakeLists.txt", + "/llama.cpp/common/parsers/sources.cmake", "/llama.cpp/ggml/CMakeLists.txt", "/llama.cpp/ggml/src/CMakeLists.txt", "/llama.cpp/src/CMakeLists.txt", "/llama.cpp/cmake", "/llama.cpp/ggml/cmake", - "/llama.cpp/common/cmake", ] [dependencies] @@ -77,6 +78,7 @@ cuda-no-vmm = ["cuda", "llama-cpp-bindings-build/cuda-no-vmm"] metal = ["llama-cpp-bindings-build/metal"] dynamic-link = ["llama-cpp-bindings-build/dynamic-link"] dynamic-backends = ["dynamic-link", "llama-cpp-bindings-build/dynamic-backends"] +fault-injection = ["llama-cpp-bindings-build/fault-injection"] vulkan = ["llama-cpp-bindings-build/vulkan"] openmp = ["llama-cpp-bindings-build/openmp"] rocm = ["llama-cpp-bindings-build/rocm"] diff --git a/llama-cpp-bindings-sys/build.rs b/llama-cpp-bindings-sys/build.rs index bae50bed6..bd7ad472f 100644 --- a/llama-cpp-bindings-sys/build.rs +++ b/llama-cpp-bindings-sys/build.rs @@ -1,3 +1,8 @@ -fn main() -> Result<(), llama_cpp_bindings_build::BuildError> { - llama_cpp_bindings_build::build() +use std::io; + +use llama_cpp_bindings_build::build_environment::BuildEnvironment; +use llama_cpp_bindings_build::build_error::BuildError; + +fn main() -> Result<(), BuildError> { + llama_cpp_bindings_build::build(&BuildEnvironment::of_process(), &mut io::stdout()) } diff --git a/llama-cpp-bindings-sys/llama.cpp b/llama-cpp-bindings-sys/llama.cpp index 5266f24da..8642541b2 160000 --- a/llama-cpp-bindings-sys/llama.cpp +++ b/llama-cpp-bindings-sys/llama.cpp @@ -1 +1 @@ -Subproject commit 5266f24da75dc449bd56cbed7addb9c8e4a6a73e +Subproject commit 8642541b255bfb5dc39cd351a178be09013a55d6 diff --git a/llama-cpp-bindings-sys/src/lib.rs b/llama-cpp-bindings-sys/src/lib.rs index 0b6b4ba15..4b3b5f2a6 100644 --- a/llama-cpp-bindings-sys/src/lib.rs +++ b/llama-cpp-bindings-sys/src/lib.rs @@ -7,12 +7,6 @@ reason = "bindgen emits C struct and enum names verbatim and they don't follow Rust naming" )] #![expect( - unpredictable_function_pointer_comparisons, - reason = "bindgen-generated FFI function pointers are opaque and the lint cannot reason about them" -)] -#![expect( - clippy::derive_partial_eq_without_eq, - clippy::doc_markdown, clippy::pub_underscore_fields, clippy::use_self, reason = "bindgen writes this file from the llama.cpp headers; its shape is not ours to change" diff --git a/llama-cpp-bindings-sys/wrapper.h b/llama-cpp-bindings-sys/wrapper.h index e8469f430..e45b1fc1b 100644 --- a/llama-cpp-bindings-sys/wrapper.h +++ b/llama-cpp-bindings-sys/wrapper.h @@ -1,13 +1,14 @@ -#include "llama.cpp/include/llama.h" -#include "llama.cpp/ggml/include/gguf.h" -#include "wrapper_chat_apply.h" -#include "wrapper_chat_parse.h" +#include +#include +#include "wrapper_batch.h" +#include "wrapper_chat_session.h" +#include "wrapper_chat_templates.h" #include "wrapper_common.h" #include "wrapper_context.h" +#include "wrapper_fault_injection.h" #include "wrapper_fit.h" #include "wrapper_gbnf.h" #include "wrapper_mtmd.h" -#include "wrapper_reasoning.h" +#include "wrapper_parsed_chat.h" #include "wrapper_sampler_init.h" #include "wrapper_state.h" -#include "wrapper_tool_calls.h" diff --git a/llama-cpp-bindings-sys/wrapper_batch.cpp b/llama-cpp-bindings-sys/wrapper_batch.cpp new file mode 100644 index 000000000..08148a471 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_batch.cpp @@ -0,0 +1,159 @@ +#include "wrapper_batch.h" +#include "wrapper_fault_injection.h" +#include "wrapper_report_current_exception.h" + +#include + +#include +#include +#include +#include + +extern "C" auto llama_rs_batch_ext_init( + struct llama_context * ctx, + struct llama_batch_ext ** out_batch, + char ** out_error) -> llama_rs_batch_ext_init_status { + if (out_batch != nullptr) { + *out_batch = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (ctx == nullptr) { + return LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG; + } + if (out_batch == nullptr) { + return LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG; + } + + try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_BATCH_EXT_INIT_THROWS); + *out_batch = llama_batch_ext_init(ctx); + + return LLAMA_RS_BATCH_EXT_INIT_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_BATCH_EXT_INIT_ERROR_STRING_ALLOCATION_FAILED); + } +} + +namespace { + +auto add_entry( + struct llama_batch_ext * batch, + const llama_rs_batch_entry & entry, + const gsl::span seq_ids, + int32_t & out_llama_cpp_return_code) -> llama_rs_batch_ext_assign_entries_status { + if (seq_ids.empty()) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS; + } + + const int32_t index = llama_batch_ext_add_token(batch, seq_ids[0], entry.token); + + if (index < 0) { + out_llama_cpp_return_code = index; + + switch (index) { + case -1: + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL; + case -2: + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN; + case -3: + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID; + default: + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE; + } + } + + const gsl::span additional_seq_ids = seq_ids.subspan(1); + const bool every_seq_id_added = std::all_of( + additional_seq_ids.begin(), additional_seq_ids.end(), [&](const llama_seq_id seq_id) -> bool { + return llama_batch_ext_add_seq(batch, index, seq_id); + }); + + if (!every_seq_id_added) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID; + } + + if (!llama_batch_ext_set_pos(batch, index, &entry.pos)) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED; + } + + if (!llama_batch_ext_set_output_logits(batch, index, entry.output)) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED; + } + + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK; +} + +} // namespace + +extern "C" auto llama_rs_batch_ext_assign_entries( + struct llama_batch_ext * batch, + const llama_rs_batch_entry * entries, + size_t entries_count, + const llama_seq_id * seq_ids, + size_t seq_ids_count, + int32_t * out_llama_cpp_return_code, + char ** out_error) -> llama_rs_batch_ext_assign_entries_status { + if (out_llama_cpp_return_code != nullptr) { + *out_llama_cpp_return_code = 0; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (batch == nullptr) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG; + } + if (entries == nullptr) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG; + } + if (seq_ids == nullptr) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG; + } + if (out_llama_cpp_return_code == nullptr) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG; + } + + try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS); + llama_batch_ext_clear(batch); + + const gsl::span entry_span(entries, entries_count); + const gsl::span seq_id_span(seq_ids, seq_ids_count); + + for (const llama_rs_batch_entry & entry : entry_span) { + if (entry.seq_ids_offset > seq_id_span.size() || + entry.seq_ids_count > seq_id_span.size() - entry.seq_ids_offset) { + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE; + } + + const llama_rs_batch_ext_assign_entries_status status = add_entry( + batch, + entry, + seq_id_span.subspan(entry.seq_ids_offset, entry.seq_ids_count), + *out_llama_cpp_return_code); + + if (status != LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK) { + return status; + } + } + + return LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED); + } +} diff --git a/llama-cpp-bindings-sys/wrapper_batch.h b/llama-cpp-bindings-sys/wrapper_batch.h new file mode 100644 index 000000000..5c3d9acac --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_batch.h @@ -0,0 +1,67 @@ +#pragma once + +#include + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum llama_rs_batch_ext_init_status { + LLAMA_RS_BATCH_EXT_INIT_OK = 0, + LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG, + LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG, + LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG, + LLAMA_RS_BATCH_EXT_INIT_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_batch_ext_init_status; + +llama_rs_batch_ext_init_status llama_rs_batch_ext_init( + struct llama_context * ctx, + struct llama_batch_ext ** out_batch, + char ** out_error); + +typedef struct llama_rs_batch_entry { + llama_token token; + llama_pos pos; + size_t seq_ids_offset; + size_t seq_ids_count; + bool output; +} llama_rs_batch_entry; + +typedef enum llama_rs_batch_ext_assign_entries_status { + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK = 0, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_batch_ext_assign_entries_status; + +llama_rs_batch_ext_assign_entries_status llama_rs_batch_ext_assign_entries( + struct llama_batch_ext * batch, + const llama_rs_batch_entry * entries, + size_t entries_count, + const llama_seq_id * seq_ids, + size_t seq_ids_count, + int32_t * out_llama_cpp_return_code, + char ** out_error); + +#ifdef __cplusplus +} +#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_apply.cpp b/llama-cpp-bindings-sys/wrapper_chat_apply.cpp deleted file mode 100644 index 35618c217..000000000 --- a/llama-cpp-bindings-sys/wrapper_chat_apply.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include "wrapper_chat_apply.h" -#include "wrapper_token_text.h" - -#include "llama.cpp/common/chat-auto-parser.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" -#include "wrapper_report_current_exception.h" -#include "wrapper_utils.h" - -#include -#include -#include -#include - -using wrapper_helpers::token_text_or_empty; - -extern "C" auto llama_rs_apply_chat_template( - const struct llama_model * model, - const char * template_src, - const char * const * roles, - const char * const * contents, - size_t n_messages, - int add_generation_prompt, - int enable_thinking, - char ** out_string, - char ** out_error) -> llama_rs_apply_chat_template_status { - if (out_string != nullptr) { - *out_string = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MODEL_ARG; - } - if (template_src == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_TEMPLATE_ARG; - } - if (n_messages > 0 && ((roles == nullptr) || (contents == nullptr))) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MESSAGES_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_STRING_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_ERROR_ARG; - } - - try { - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(template_src, bos_token, eos_token); - - common_json messages = common_json::array(); - const gsl::span role_span(roles, n_messages); - const gsl::span content_span(contents, n_messages); - for (size_t index = 0; index < n_messages; index++) { - messages.push_back({ - { "role", (role_span[index] != nullptr) ? role_span[index] : "" }, - { "content", (content_span[index] != nullptr) ? content_span[index] : "" }, - }); - } - - autoparser::generation_params inputs; - inputs.messages = std::move(messages); - inputs.tools = common_json::array(); - inputs.add_generation_prompt = add_generation_prompt != 0; - inputs.enable_thinking = enable_thinking != 0; - - std::string const rendered = common_chat_template_direct_apply(tmpl, inputs); - if (rendered.empty()) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_TEMPLATE_APPLICATION_FAILED; - } - - *out_string = llama_rs_dup_string(rendered); - if (*out_string == nullptr) { - return LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED; - } - - return LLAMA_RS_APPLY_CHAT_TEMPLATE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED); - } -} diff --git a/llama-cpp-bindings-sys/wrapper_chat_apply.h b/llama-cpp-bindings-sys/wrapper_chat_apply.h deleted file mode 100644 index 3add989a3..000000000 --- a/llama-cpp-bindings-sys/wrapper_chat_apply.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" -#include "wrapper_utils.h" - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum llama_rs_apply_chat_template_status { - LLAMA_RS_APPLY_CHAT_TEMPLATE_OK = 0, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MODEL_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_TEMPLATE_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MESSAGES_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_STRING_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_ERROR_ARG, - LLAMA_RS_APPLY_CHAT_TEMPLATE_MODEL_HAS_NO_VOCAB, - LLAMA_RS_APPLY_CHAT_TEMPLATE_TEMPLATE_APPLICATION_FAILED, - LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_apply_chat_template_status; - -llama_rs_apply_chat_template_status llama_rs_apply_chat_template( - const struct llama_model * model, - const char * template_src, - const char * const * roles, - const char * const * contents, - size_t n_messages, - int add_generation_prompt, - int enable_thinking, - char ** out_string, - char ** out_error); - -#ifdef __cplusplus -} -#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_parse.cpp b/llama-cpp-bindings-sys/wrapper_chat_parse.cpp deleted file mode 100644 index b90302f7a..000000000 --- a/llama-cpp-bindings-sys/wrapper_chat_parse.cpp +++ /dev/null @@ -1,446 +0,0 @@ -#include "wrapper_chat_parse.h" -#include "peg-parser.h" -#include "wrapper_token_text.h" - -#include "llama.cpp/common/chat-auto-parser.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" -#include "wrapper_report_current_exception.h" -#include "wrapper_utils.h" - -#include -#include -#include -#include - -using wrapper_helpers::token_text_or_empty; - -struct llama_rs_parsed_chat { - common_chat_msg message; -}; - -struct llama_rs_chat_parser { - autoparser::autoparser parser; -}; - -struct llama_rs_chat_tools_parser { - common_peg_arena arena; -}; - -namespace { -void dup_or_set_alloc_flag(const std::string & source, char ** out_dup, bool * out_alloc_failed) { - *out_dup = llama_rs_dup_string(source); - *out_alloc_failed = (*out_dup == nullptr); -} - -auto parse_with_tools_parser( - const common_peg_arena & chat_parser, - const char * input, - int is_partial, - llama_rs_parsed_chat_handle * out_handle, - char ** out_error) -> llama_rs_parse_chat_message_status { - try { - common_chat_parser_params parser_params; - parser_params.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - - common_chat_msg parsed = - common_chat_peg_parse(chat_parser, input, is_partial != 0, parser_params); - - auto handle = std::make_unique(); - handle->message = std::move(parsed); - - *out_handle = handle.release(); - - return LLAMA_RS_PARSE_CHAT_MESSAGE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSE_CHAT_MESSAGE_ERROR_STRING_ALLOCATION_FAILED); - } -} -} // namespace - -extern "C" auto llama_rs_chat_parser_create( - const struct llama_model * model, - llama_rs_chat_parser_handle * out_parser, - char ** out_error) -> llama_rs_chat_parser_create_status { - if (out_parser != nullptr) { - *out_parser = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_NULL_MODEL_ARG; - } - if (out_parser == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_PARSER_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_ERROR_ARG; - } - - try { - const char * tmpl_src = llama_model_chat_template(model, nullptr); - if (tmpl_src == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_CHAT_TEMPLATE; - } - - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(tmpl_src, bos_token, eos_token); - - auto parser_handle = std::make_unique(); - parser_handle->parser.analyze_template(tmpl); - - *out_parser = parser_handle.release(); - - return LLAMA_RS_CHAT_PARSER_CREATE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_chat_parser_free( - llama_rs_chat_parser_handle parser, - char ** out_error) -> llama_rs_chat_parser_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(parser); - return LLAMA_RS_CHAT_PARSER_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_chat_tools_parser_create( - llama_rs_chat_parser_handle parser, - const char * tools_json, - llama_rs_chat_tools_parser_handle * out_tools_parser, - char ** out_error) -> llama_rs_chat_tools_parser_create_status { - if (out_tools_parser != nullptr) { - *out_tools_parser = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (parser == nullptr) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG; - } - if (tools_json == nullptr) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG; - } - if (out_tools_parser == nullptr) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG; - } - - try { - autoparser::generation_params inputs; - - inputs.tools = common_json::parse(tools_json); - - if (!inputs.tools.is_array()) { - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY; - } - - auto tools_parser_handle = std::make_unique(); - tools_parser_handle->arena = parser->parser.build_parser(inputs, std::string()); - - *out_tools_parser = tools_parser_handle.release(); - - return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_chat_tools_parser_free( - llama_rs_chat_tools_parser_handle tools_parser, - char ** out_error) -> llama_rs_chat_tools_parser_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(tools_parser); - return LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parse_chat_message( - llama_rs_chat_tools_parser_handle tools_parser, - const char * input, - int is_partial, - llama_rs_parsed_chat_handle * out_handle, - char ** out_error) -> llama_rs_parse_chat_message_status { - if (out_handle != nullptr) { - *out_handle = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (tools_parser == nullptr) { - return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG; - } - if (input == nullptr) { - return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG; - } - if (out_handle == nullptr) { - return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_HANDLE_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG; - } - - return parse_with_tools_parser(tools_parser->arena, input, is_partial, out_handle, out_error); -} - -extern "C" auto llama_rs_parsed_chat_free( - llama_rs_parsed_chat_handle handle, - char ** out_error) -> llama_rs_parsed_chat_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(handle); - return LLAMA_RS_PARSED_CHAT_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_tool_call_count( - llama_rs_parsed_chat_handle handle, - size_t * out_count, - char ** out_error) -> llama_rs_parsed_chat_tool_call_count_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_count != nullptr) { - *out_count = 0; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_HANDLE_ARG; - } - if (out_count == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_OUT_COUNT_ARG; - } - try { - *out_count = handle->message.tool_calls.size(); - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_tool_call_id( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_tool_call_id_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_OUT_STRING_ARG; - } - try { - if (index >= handle->message.tool_calls.size()) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_INDEX_OUT_OF_BOUNDS; - } - bool alloc_failed = false; - dup_or_set_alloc_flag(handle->message.tool_calls[index].id, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_tool_call_name( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_tool_call_name_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_OUT_STRING_ARG; - } - try { - if (index >= handle->message.tool_calls.size()) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_INDEX_OUT_OF_BOUNDS; - } - bool alloc_failed = false; - dup_or_set_alloc_flag(handle->message.tool_calls[index].name, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_tool_call_arguments( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_tool_call_arguments_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_OUT_STRING_ARG; - } - try { - if (index >= handle->message.tool_calls.size()) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_INDEX_OUT_OF_BOUNDS; - } - bool alloc_failed = false; - dup_or_set_alloc_flag( - handle->message.tool_calls[index].arguments, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_content( - llama_rs_parsed_chat_handle handle, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_content_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_CONTENT_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_CONTENT_NULL_OUT_STRING_ARG; - } - try { - bool alloc_failed = false; - dup_or_set_alloc_flag(handle->message.content, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_CONTENT_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_parsed_chat_reasoning_content( - llama_rs_parsed_chat_handle handle, - char ** out_string, - char ** out_error) -> llama_rs_parsed_chat_reasoning_content_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_string != nullptr) { - *out_string = nullptr; - } - if (handle == nullptr) { - return LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_HANDLE_ARG; - } - if (out_string == nullptr) { - return LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_OUT_STRING_ARG; - } - try { - bool alloc_failed = false; - dup_or_set_alloc_flag(handle->message.reasoning_content, out_string, &alloc_failed); - if (alloc_failed) { - return LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED; - } - return LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED); - } -} diff --git a/llama-cpp-bindings-sys/wrapper_chat_parse.h b/llama-cpp-bindings-sys/wrapper_chat_parse.h deleted file mode 100644 index 181d1efb7..000000000 --- a/llama-cpp-bindings-sys/wrapper_chat_parse.h +++ /dev/null @@ -1,199 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" -#include "wrapper_utils.h" - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -struct llama_rs_parsed_chat; -typedef struct llama_rs_parsed_chat * llama_rs_parsed_chat_handle; - -struct llama_rs_chat_parser; -typedef struct llama_rs_chat_parser * llama_rs_chat_parser_handle; - -struct llama_rs_chat_tools_parser; -typedef struct llama_rs_chat_tools_parser * llama_rs_chat_tools_parser_handle; - -typedef enum llama_rs_chat_parser_create_status { - LLAMA_RS_CHAT_PARSER_CREATE_OK = 0, - LLAMA_RS_CHAT_PARSER_CREATE_NULL_MODEL_ARG, - LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_PARSER_ARG, - LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_ERROR_ARG, - LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_CHAT_TEMPLATE, - LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_VOCAB, - LLAMA_RS_CHAT_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_chat_parser_create_status; - -llama_rs_chat_parser_create_status llama_rs_chat_parser_create( - const struct llama_model * model, - llama_rs_chat_parser_handle * out_parser, - char ** out_error); - -typedef enum llama_rs_chat_parser_free_status { - LLAMA_RS_CHAT_PARSER_FREE_OK = 0, - LLAMA_RS_CHAT_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_chat_parser_free_status; - -llama_rs_chat_parser_free_status llama_rs_chat_parser_free( - llama_rs_chat_parser_handle parser, - char ** out_error); - -typedef enum llama_rs_chat_tools_parser_create_status { - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK = 0, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_chat_tools_parser_create_status; - -llama_rs_chat_tools_parser_create_status llama_rs_chat_tools_parser_create( - llama_rs_chat_parser_handle parser, - const char * tools_json, - llama_rs_chat_tools_parser_handle * out_tools_parser, - char ** out_error); - -typedef enum llama_rs_chat_tools_parser_free_status { - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK = 0, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_chat_tools_parser_free_status; - -llama_rs_chat_tools_parser_free_status llama_rs_chat_tools_parser_free( - llama_rs_chat_tools_parser_handle tools_parser, - char ** out_error); - -typedef enum llama_rs_parse_chat_message_status { - LLAMA_RS_PARSE_CHAT_MESSAGE_OK = 0, - LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG, - LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG, - LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_HANDLE_ARG, - LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG, - LLAMA_RS_PARSE_CHAT_MESSAGE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parse_chat_message_status; - -llama_rs_parse_chat_message_status llama_rs_parse_chat_message( - llama_rs_chat_tools_parser_handle tools_parser, - const char * input, - int is_partial, - llama_rs_parsed_chat_handle * out_handle, - char ** out_error); - -typedef enum llama_rs_parsed_chat_free_status { - LLAMA_RS_PARSED_CHAT_FREE_OK = 0, - LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_free_status; - -llama_rs_parsed_chat_free_status llama_rs_parsed_chat_free( - llama_rs_parsed_chat_handle handle, - char ** out_error); - -typedef enum llama_rs_parsed_chat_tool_call_count_status { - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_OK = 0, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_OUT_COUNT_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_tool_call_count_status; - -llama_rs_parsed_chat_tool_call_count_status llama_rs_parsed_chat_tool_call_count( - llama_rs_parsed_chat_handle handle, - size_t * out_count, - char ** out_error); - -typedef enum llama_rs_parsed_chat_tool_call_id_status { - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_OK = 0, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_INDEX_OUT_OF_BOUNDS, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_tool_call_id_status; - -llama_rs_parsed_chat_tool_call_id_status llama_rs_parsed_chat_tool_call_id( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error); - -typedef enum llama_rs_parsed_chat_tool_call_name_status { - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_OK = 0, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_INDEX_OUT_OF_BOUNDS, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_tool_call_name_status; - -llama_rs_parsed_chat_tool_call_name_status llama_rs_parsed_chat_tool_call_name( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error); - -typedef enum llama_rs_parsed_chat_tool_call_arguments_status { - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_OK = 0, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_INDEX_OUT_OF_BOUNDS, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_tool_call_arguments_status; - -llama_rs_parsed_chat_tool_call_arguments_status llama_rs_parsed_chat_tool_call_arguments( - llama_rs_parsed_chat_handle handle, - size_t index, - char ** out_string, - char ** out_error); - -typedef enum llama_rs_parsed_chat_content_status { - LLAMA_RS_PARSED_CHAT_CONTENT_OK = 0, - LLAMA_RS_PARSED_CHAT_CONTENT_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_CONTENT_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_content_status; - -llama_rs_parsed_chat_content_status llama_rs_parsed_chat_content( - llama_rs_parsed_chat_handle handle, - char ** out_string, - char ** out_error); - -typedef enum llama_rs_parsed_chat_reasoning_content_status { - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_OK = 0, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_HANDLE_ARG, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_OUT_STRING_ARG, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_parsed_chat_reasoning_content_status; - -llama_rs_parsed_chat_reasoning_content_status llama_rs_parsed_chat_reasoning_content( - llama_rs_parsed_chat_handle handle, - char ** out_string, - char ** out_error); - -#ifdef __cplusplus -} -#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.cpp b/llama-cpp-bindings-sys/wrapper_chat_session.cpp new file mode 100644 index 000000000..abe55c4d8 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_session.cpp @@ -0,0 +1,530 @@ +#include "wrapper_chat_session.h" +#include "wrapper_fault_injection.h" +#include "wrapper_chat_templates.h" +#include "wrapper_chat_templates_handle.h" +#include "wrapper_parsed_chat.h" +#include "wrapper_parsed_chat_handle.h" +#include "wrapper_report_current_exception.h" +#include "wrapper_utils.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct llama_rs_chat_session { + common_chat_session session; + common_chat_input generation_prompt; + std::vector generation_prompt_tokens; + common_params_sampling sampling; + std::vector preserved_tokens; + std::unordered_set trigger_rules; +}; + +struct llama_rs_chat_spans { + size_t parsed_length = 0; + std::vector spans; +}; + +namespace { +auto tool_choice_of(llama_rs_chat_tool_choice tool_choice) -> common_chat_tool_choice { + switch (tool_choice) { + case LLAMA_RS_CHAT_TOOL_CHOICE_NONE: + return COMMON_CHAT_TOOL_CHOICE_NONE; + case LLAMA_RS_CHAT_TOOL_CHOICE_REQUIRED: + return COMMON_CHAT_TOOL_CHOICE_REQUIRED; + case LLAMA_RS_CHAT_TOOL_CHOICE_AUTO: + return COMMON_CHAT_TOOL_CHOICE_AUTO; + } + + throw std::invalid_argument("unrecognized llama_rs_chat_tool_choice value"); +} + +auto continuation_of(llama_rs_chat_assistant_turn assistant_turn) -> common_chat_continuation { + switch (assistant_turn) { + case LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_REASONING: + return COMMON_CHAT_CONTINUATION_REASONING; + case LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_CONTENT: + return COMMON_CHAT_CONTINUATION_CONTENT; + case LLAMA_RS_CHAT_ASSISTANT_TURN_OMITTED: + case LLAMA_RS_CHAT_ASSISTANT_TURN_NEW: + return COMMON_CHAT_CONTINUATION_NONE; + } + + throw std::invalid_argument("unrecognized llama_rs_chat_assistant_turn value"); +} + +auto span_kind_of_tag(const std::string & tag) -> std::optional { + if (tag == common_chat_peg_builder::REASONING_BLOCK) { + return LLAMA_RS_CHAT_SPAN_KIND_REASONING_BLOCK; + } + if (tag == common_chat_peg_builder::REASONING) { + return LLAMA_RS_CHAT_SPAN_KIND_REASONING; + } + if (tag == common_chat_peg_builder::CONTENT) { + return LLAMA_RS_CHAT_SPAN_KIND_CONTENT; + } + if (tag == common_chat_peg_builder::TOOL_OPEN || tag == common_chat_peg_builder::TOOL_CLOSE || + tag == common_chat_peg_builder::TOOL_ARG_OPEN || tag == common_chat_peg_builder::TOOL_ARG_CLOSE) { + return LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER; + } + if (tag == common_chat_peg_builder::TOOL || tag == common_chat_peg_builder::TOOL_ID || + tag == common_chat_peg_builder::TOOL_NAME || tag == common_chat_peg_builder::TOOL_ARGS || + tag == common_chat_peg_builder::TOOL_ARG || tag == common_chat_peg_builder::TOOL_ARG_NAME || + tag == common_chat_peg_builder::TOOL_ARG_VALUE || tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE || + tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT || tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY || + tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ITEM) { + return LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL; + } + + return std::nullopt; +} + +auto trigger_rules_of(const common_peg_arena & parser) -> std::unordered_set { + std::unordered_set trigger_rules; + + for (common_peg_parser_id parser_id = 0; parser_id < parser.size(); ++parser_id) { + const auto * rule_parser = std::get_if(&parser.get(parser_id)); + if (rule_parser != nullptr && rule_parser->trigger && !rule_parser->name.empty() && + parser.get_rule(rule_parser->name) == parser_id) { + trigger_rules.insert(rule_parser->name); + } + } + + return trigger_rules; +} + +auto span_kind_of_node(const std::unordered_set & trigger_rules, const common_peg_ast_node & node) + -> std::optional { + if (const auto kind = span_kind_of_tag(node.tag)) { + return kind; + } + if (trigger_rules.find(node.rule) != trigger_rules.end()) { + return LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER; + } + + return std::nullopt; +} + +auto chat_input_of( + const common_chat_input & prefix, + const char * text, + size_t text_length, + const llama_token * byte_tokens) -> common_chat_input { + const gsl::span tokens(byte_tokens, text_length); + common_chat_input input; + input.text.reserve(prefix.text.size() + text_length); + input.text.append(prefix.text).append(text, text_length); + input.tokens.reserve(prefix.tokens.size() + text_length); + input.tokens.insert(input.tokens.end(), prefix.tokens.begin(), prefix.tokens.end()); + input.tokens.insert(input.tokens.end(), tokens.begin(), tokens.end()); + + return input; +} + +auto parse_flags_of(bool is_partial) -> common_peg_parse_flags { + return is_partial ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_COMPLETE; +} +} // namespace + +extern "C" auto llama_rs_chat_session_create( + llama_rs_chat_templates_handle templates, + const struct llama_vocab * vocab, + const char * messages_json, + const char * tools_json, + llama_rs_chat_session_options options, + llama_rs_chat_session_handle * out_session, + char ** out_error) -> llama_rs_chat_session_create_status { + if (out_session != nullptr) { + *out_session = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (templates == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_TEMPLATES_ARG; + } + if (vocab == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_VOCAB_ARG; + } + if (messages_json == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_MESSAGES_JSON_ARG; + } + if (tools_json == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG; + } + if (out_session == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG; + } + + try { + common_chat_templates_inputs inputs; + + inputs.messages = common_chat_msgs_parse_oaicompat(common_json::parse(messages_json)); + + try { + inputs.tools = common_chat_tools_parse_oaicompat(common_json::parse(tools_json)); + } catch (const common_json_error & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } catch (const std::runtime_error & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } + + inputs.add_generation_prompt = options.assistant_turn == LLAMA_RS_CHAT_ASSISTANT_TURN_NEW; + inputs.continue_final_message = continuation_of(options.assistant_turn); + inputs.enable_thinking = options.enable_thinking; + inputs.parallel_tool_calls = options.parallel_tool_calls; + inputs.tool_choice = tool_choice_of(options.tool_choice); + inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO; + + auto chat_session = std::make_unique(); + + try { + chat_session->session = common_chat_session(templates->templates.get(), vocab, inputs); + } catch (const std::bad_alloc &) { + throw; + } catch (const std::exception & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } + + chat_session->generation_prompt = common_chat_input_tokenize(vocab, chat_session->session.generation_prompt()); + for (const llama_token token : chat_session->generation_prompt.tokens) { + if (token != LLAMA_TOKEN_NULL) { + chat_session->generation_prompt_tokens.push_back(token); + } + } + chat_session->session.apply_sampling(chat_session->sampling); + chat_session->preserved_tokens.assign( + chat_session->sampling.preserved_tokens.begin(), chat_session->sampling.preserved_tokens.end()); + chat_session->trigger_rules = trigger_rules_of(chat_session->session.parser()); + + *out_session = chat_session.release(); + + return LLAMA_RS_CHAT_SESSION_CREATE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_session_free(llama_rs_chat_session_handle session) -> void { + std::unique_ptr(session).reset(); +} + +extern "C" auto llama_rs_chat_session_prompt_part_count(llama_rs_chat_session_handle session) -> size_t { + return session->session.prompt().parts.size(); +} + +extern "C" auto llama_rs_chat_session_prompt_part_text( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_CHAT_SESSION_PROMPT_PART_TEXT_NOT_UTF8, + session->session.prompt().parts.at(index).text, + out_length); +} + +extern "C" auto llama_rs_chat_session_prompt_part_is_input(llama_rs_chat_session_handle session, size_t index) + -> bool { + return session->session.prompt().parts.at(index).is_input; +} + +extern "C" auto llama_rs_chat_session_grammar(llama_rs_chat_session_handle session, size_t * out_length) + -> const char * { + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8, session->session.grammar(), out_length); +} + +extern "C" auto llama_rs_chat_session_grammar_lazy(llama_rs_chat_session_handle session) -> bool { + return session->sampling.grammar_lazy; +} + +extern "C" auto llama_rs_chat_session_grammar_trigger_count(llama_rs_chat_session_handle session) -> size_t { + return session->sampling.grammar_triggers.size(); +} + +extern "C" auto llama_rs_chat_session_grammar_trigger_type(llama_rs_chat_session_handle session, size_t index) + -> llama_rs_chat_grammar_trigger_type { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED)) { + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED; + } + + switch (session->sampling.grammar_triggers.at(index).type) { + case COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN: + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN; + case COMMON_GRAMMAR_TRIGGER_TYPE_WORD: + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD; + case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN: + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN; + case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL: + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL; + } + + return LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED; +} + +extern "C" auto llama_rs_chat_session_grammar_trigger_value( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_VALUE_NOT_UTF8, + session->sampling.grammar_triggers.at(index).value, + out_length); +} + +extern "C" auto llama_rs_chat_session_grammar_trigger_token(llama_rs_chat_session_handle session, size_t index) + -> llama_token { + return session->sampling.grammar_triggers.at(index).token; +} + +extern "C" auto llama_rs_chat_session_preserved_token_count(llama_rs_chat_session_handle session) -> size_t { + return session->preserved_tokens.size(); +} + +extern "C" auto llama_rs_chat_session_preserved_token(llama_rs_chat_session_handle session, size_t index) + -> llama_token { + return session->preserved_tokens.at(index); +} + +extern "C" auto llama_rs_chat_session_generation_prompt(llama_rs_chat_session_handle session, size_t * out_length) + -> const char * { + return llama_rs_string_view(session->session.generation_prompt(), out_length); +} + +extern "C" auto llama_rs_chat_session_generation_prompt_token_count(llama_rs_chat_session_handle session) -> size_t { + return session->generation_prompt_tokens.size(); +} + +extern "C" auto llama_rs_chat_session_generation_prompt_token(llama_rs_chat_session_handle session, size_t index) + -> llama_token { + return session->generation_prompt_tokens.at(index); +} + +extern "C" auto llama_rs_chat_session_thinking_start_tag(llama_rs_chat_session_handle session, size_t * out_length) + -> const char * { + return llama_rs_string_view(session->session.thinking_start_tag(), out_length); +} + +extern "C" auto llama_rs_chat_session_thinking_end_tag_count(llama_rs_chat_session_handle session) -> size_t { + return session->session.thinking_end_tags().size(); +} + +extern "C" auto llama_rs_chat_session_thinking_end_tag( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_string_view(session->session.thinking_end_tags().at(index), out_length); +} + +extern "C" auto llama_rs_chat_session_parse( + llama_rs_chat_session_handle session, + const char * text, + size_t text_length, + const llama_token * byte_tokens, + bool is_partial, + llama_rs_parsed_chat_handle * out_handle, + char ** out_error) -> llama_rs_chat_session_parse_status { + if (out_handle != nullptr) { + *out_handle = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (session == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG; + } + if (text == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG; + } + if (byte_tokens == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG; + } + if (out_handle == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG; + } + + try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS); + + common_chat_parser_params parser_params; + parser_params.format = session->session.format(); + parser_params.generation_prompt = session->generation_prompt; + + auto parsed = std::make_unique(); + + try { + parsed->message = common_chat_peg_parse( + session->session.parser(), + chat_input_of(common_chat_input(), text, text_length, byte_tokens), + is_partial, + parser_params); + } catch (const std::bad_alloc &) { + throw; + } catch (const std::runtime_error & err) { + return llama_rs_report_exception_message( + out_error, err.what(), LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, + LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED); + } + + *out_handle = parsed.release(); + + return LLAMA_RS_CHAT_SESSION_PARSE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_session_classify( + llama_rs_chat_session_handle session, + const char * text, + size_t text_length, + const llama_token * byte_tokens, + bool is_partial, + llama_rs_chat_spans_handle * out_spans, + char ** out_error) -> llama_rs_chat_session_classify_status { + if (out_spans != nullptr) { + *out_spans = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (session == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG; + } + if (text == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG; + } + if (byte_tokens == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG; + } + if (out_spans == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG; + } + + try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + + common_chat_input parse_input = chat_input_of(session->generation_prompt, text, text_length, byte_tokens); + const size_t generation_prompt_length = session->generation_prompt.size(); + + common_peg_parse_context context( + std::move(parse_input.text), std::move(parse_input.tokens), parse_flags_of(is_partial)); + const common_peg_parse_result result = session->session.parser().parse(context); + + if (result.fail() && (!is_partial || result.end == 0)) { + return LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT; + } + + auto spans = std::make_unique(); + spans->parsed_length = result.end > generation_prompt_length ? result.end - generation_prompt_length : 0; + + context.ast.visit(result, [&](const common_peg_ast_node & node) -> void { + const auto kind = span_kind_of_node(session->trigger_rules, node); + if (!kind || node.end <= generation_prompt_length) { + return; + } + spans->spans.push_back({ + *kind, + node.start > generation_prompt_length ? node.start - generation_prompt_length : 0, + node.end - generation_prompt_length, + }); + }); + + *out_spans = spans.release(); + + return LLAMA_RS_CHAT_SESSION_CLASSIFY_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_spans_view_of(llama_rs_chat_spans_handle spans) -> llama_rs_chat_spans_view { + return {spans->spans.data(), spans->spans.size(), spans->parsed_length}; +} + +extern "C" auto llama_rs_chat_spans_free(llama_rs_chat_spans_handle spans) -> void { + std::unique_ptr(spans).reset(); +} + +extern "C" auto llama_rs_regex_escape( + const char * text, + size_t text_length, + char ** out_string, + char ** out_error) -> llama_rs_regex_escape_status { + if (out_string != nullptr) { + *out_string = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (text == nullptr) { + return LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG; + } + if (out_string == nullptr) { + return LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG; + } + + try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS); + + *out_string = llama_rs_dup_string(regex_escape(std::string(text, text_length))); + if (*out_string == nullptr) { + return LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED; + } + + return LLAMA_RS_REGEX_ESCAPE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED); + } +} diff --git a/llama-cpp-bindings-sys/wrapper_chat_session.h b/llama-cpp-bindings-sys/wrapper_chat_session.h new file mode 100644 index 000000000..59550e8dc --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_session.h @@ -0,0 +1,207 @@ +#pragma once + +#include +#include "wrapper_parsed_chat.h" +#include "wrapper_chat_templates.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct llama_rs_chat_session; +typedef struct llama_rs_chat_session * llama_rs_chat_session_handle; + +struct llama_rs_chat_spans; +typedef struct llama_rs_chat_spans * llama_rs_chat_spans_handle; + +typedef enum llama_rs_chat_tool_choice { + LLAMA_RS_CHAT_TOOL_CHOICE_AUTO = 0, + LLAMA_RS_CHAT_TOOL_CHOICE_NONE, + LLAMA_RS_CHAT_TOOL_CHOICE_REQUIRED, +} llama_rs_chat_tool_choice; + +typedef enum llama_rs_chat_assistant_turn { + LLAMA_RS_CHAT_ASSISTANT_TURN_OMITTED = 0, + LLAMA_RS_CHAT_ASSISTANT_TURN_NEW, + LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_REASONING, + LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_CONTENT, +} llama_rs_chat_assistant_turn; + +typedef struct llama_rs_chat_session_options { + llama_rs_chat_assistant_turn assistant_turn; + bool enable_thinking; + bool parallel_tool_calls; + llama_rs_chat_tool_choice tool_choice; +} llama_rs_chat_session_options; + +typedef enum llama_rs_chat_session_create_status { + LLAMA_RS_CHAT_SESSION_CREATE_OK = 0, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_TEMPLATES_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_VOCAB_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_MESSAGES_JSON_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, + LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, + LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, + LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_session_create_status; + +llama_rs_chat_session_create_status llama_rs_chat_session_create( + llama_rs_chat_templates_handle templates, + const struct llama_vocab * vocab, + const char * messages_json, + const char * tools_json, + llama_rs_chat_session_options options, + llama_rs_chat_session_handle * out_session, + char ** out_error); + +void llama_rs_chat_session_free(llama_rs_chat_session_handle session); + +size_t llama_rs_chat_session_prompt_part_count(llama_rs_chat_session_handle session); + +const char * llama_rs_chat_session_prompt_part_text( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length); + +bool llama_rs_chat_session_prompt_part_is_input(llama_rs_chat_session_handle session, size_t index); + +const char * llama_rs_chat_session_grammar(llama_rs_chat_session_handle session, size_t * out_length); + +bool llama_rs_chat_session_grammar_lazy(llama_rs_chat_session_handle session); + +typedef enum llama_rs_chat_grammar_trigger_type { + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN = 0, + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD, + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN, + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, + LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED, +} llama_rs_chat_grammar_trigger_type; + +size_t llama_rs_chat_session_grammar_trigger_count(llama_rs_chat_session_handle session); + +llama_rs_chat_grammar_trigger_type llama_rs_chat_session_grammar_trigger_type( + llama_rs_chat_session_handle session, + size_t index); + +const char * llama_rs_chat_session_grammar_trigger_value( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length); + +llama_token llama_rs_chat_session_grammar_trigger_token(llama_rs_chat_session_handle session, size_t index); + +size_t llama_rs_chat_session_preserved_token_count(llama_rs_chat_session_handle session); + +llama_token llama_rs_chat_session_preserved_token(llama_rs_chat_session_handle session, size_t index); + +const char * llama_rs_chat_session_generation_prompt(llama_rs_chat_session_handle session, size_t * out_length); + +size_t llama_rs_chat_session_generation_prompt_token_count(llama_rs_chat_session_handle session); + +llama_token llama_rs_chat_session_generation_prompt_token(llama_rs_chat_session_handle session, size_t index); + +const char * llama_rs_chat_session_thinking_start_tag(llama_rs_chat_session_handle session, size_t * out_length); + +size_t llama_rs_chat_session_thinking_end_tag_count(llama_rs_chat_session_handle session); + +const char * llama_rs_chat_session_thinking_end_tag( + llama_rs_chat_session_handle session, + size_t index, + size_t * out_length); + +typedef enum llama_rs_chat_session_parse_status { + LLAMA_RS_CHAT_SESSION_PARSE_OK = 0, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, + LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_session_parse_status; + +llama_rs_chat_session_parse_status llama_rs_chat_session_parse( + llama_rs_chat_session_handle session, + const char * text, + size_t text_length, + const llama_token * byte_tokens, + bool is_partial, + llama_rs_parsed_chat_handle * out_handle, + char ** out_error); + +typedef enum llama_rs_chat_span_kind { + LLAMA_RS_CHAT_SPAN_KIND_REASONING_BLOCK = 0, + LLAMA_RS_CHAT_SPAN_KIND_REASONING, + LLAMA_RS_CHAT_SPAN_KIND_CONTENT, + LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL, + LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER, +} llama_rs_chat_span_kind; + +typedef struct llama_rs_chat_span { + llama_rs_chat_span_kind kind; + size_t start; + size_t end; +} llama_rs_chat_span; + +typedef struct llama_rs_chat_spans_view { + const llama_rs_chat_span * spans; + size_t spans_count; + size_t parsed_length; +} llama_rs_chat_spans_view; + +typedef enum llama_rs_chat_session_classify_status { + LLAMA_RS_CHAT_SESSION_CLASSIFY_OK = 0, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT, + LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_session_classify_status; + +llama_rs_chat_session_classify_status llama_rs_chat_session_classify( + llama_rs_chat_session_handle session, + const char * text, + size_t text_length, + const llama_token * byte_tokens, + bool is_partial, + llama_rs_chat_spans_handle * out_spans, + char ** out_error); + +llama_rs_chat_spans_view llama_rs_chat_spans_view_of(llama_rs_chat_spans_handle spans); + +void llama_rs_chat_spans_free(llama_rs_chat_spans_handle spans); + +typedef enum llama_rs_regex_escape_status { + LLAMA_RS_REGEX_ESCAPE_OK = 0, + LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG, + LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG, + LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG, + LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_regex_escape_status; + +llama_rs_regex_escape_status llama_rs_regex_escape( + const char * text, + size_t text_length, + char ** out_string, + char ** out_error); + +#ifdef __cplusplus +} +#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.cpp b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp new file mode 100644 index 000000000..4aa9f6982 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.cpp @@ -0,0 +1,56 @@ +#include "wrapper_chat_templates.h" +#include "wrapper_chat_templates_handle.h" +#include "wrapper_report_current_exception.h" + +#include +#include + +#include +#include + +extern "C" auto llama_rs_chat_templates_init( + const struct llama_model * model, + const char * template_override, + llama_rs_chat_templates_handle * out_templates, + char ** out_error) -> llama_rs_chat_templates_init_status { + if (out_templates != nullptr) { + *out_templates = nullptr; + } + if (out_error != nullptr) { + *out_error = nullptr; + } + if (model == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG; + } + if (template_override == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_TEMPLATE_OVERRIDE_ARG; + } + if (out_templates == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG; + } + if (out_error == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG; + } + if (*template_override == '\0' && llama_model_chat_template(model, nullptr) == nullptr) { + return LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE; + } + + try { + auto templates = std::make_unique(); + templates->templates = common_chat_templates_init(model, std::string(template_override)); + + *out_templates = templates.release(); + + return LLAMA_RS_CHAT_TEMPLATES_INIT_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_CHAT_TEMPLATES_INIT_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_chat_templates_free(llama_rs_chat_templates_handle templates) -> void { + std::unique_ptr(templates).reset(); +} diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates.h b/llama-cpp-bindings-sys/wrapper_chat_templates.h new file mode 100644 index 000000000..1c343dc87 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_templates.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct llama_rs_chat_templates; +typedef struct llama_rs_chat_templates * llama_rs_chat_templates_handle; + +typedef enum llama_rs_chat_templates_init_status { + LLAMA_RS_CHAT_TEMPLATES_INIT_OK = 0, + LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG, + LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_TEMPLATE_OVERRIDE_ARG, + LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG, + LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG, + LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE, + LLAMA_RS_CHAT_TEMPLATES_INIT_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_chat_templates_init_status; + +llama_rs_chat_templates_init_status llama_rs_chat_templates_init( + const struct llama_model * model, + const char * template_override, + llama_rs_chat_templates_handle * out_templates, + char ** out_error); + +void llama_rs_chat_templates_free(llama_rs_chat_templates_handle templates); + +#ifdef __cplusplus +} +#endif diff --git a/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h b/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h new file mode 100644 index 000000000..8b7723ff7 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_chat_templates_handle.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +struct llama_rs_chat_templates { + common_chat_templates_ptr templates; +}; diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index cf653e7da..57e23a535 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -1,4 +1,5 @@ #include "wrapper_common.h" +#include "wrapper_fault_injection.h" #include #include @@ -8,15 +9,15 @@ #include #include -#include "llama.cpp/common/json-schema-to-grammar.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" +#include +#include +#include +#include #include "wrapper_report_current_exception.h" #include "wrapper_utils.h" extern "C" auto llama_rs_json_schema_to_grammar( const char * schema_json, - bool force_gbnf, char ** out_grammar, char ** out_error) -> llama_rs_json_schema_to_grammar_status { if (out_grammar != nullptr) { @@ -37,12 +38,18 @@ extern "C" auto llama_rs_json_schema_to_grammar( try { const auto schema = common_json::parse(schema_json); - const auto grammar = json_schema_to_grammar(schema, force_gbnf); + const auto grammar = json_schema_to_grammar(schema); *out_grammar = llama_rs_dup_string(grammar); if (*out_grammar == nullptr) { return LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_ERROR_STRING_ALLOCATION_FAILED; } return LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_OK; + } catch (const common_json_error & err) { + return llama_rs_report_exception_message( + out_error, + err.what(), + LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_JSON, + LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_ERROR_STRING_ALLOCATION_FAILED); } catch (const std::invalid_argument & err) { return llama_rs_report_exception_message( out_error, @@ -88,7 +95,10 @@ extern "C" auto llama_rs_sampler_init_grammar( return LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_ERROR_ARG; } try { - *out_sampler = llama_sampler_init_grammar(vocab, grammar_str, grammar_root); + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_THROWS); + *out_sampler = llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_RETURNS_NULL) + ? nullptr + : llama_sampler_init_grammar(vocab, grammar_str, grammar_root); if (*out_sampler == nullptr) { return LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_RETURNED_NULL; } @@ -125,6 +135,7 @@ extern "C" auto llama_rs_sampler_init_grammar_lazy_patterns( return LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG; } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_THROWS); *out_sampler = llama_sampler_init_grammar_lazy_patterns( vocab, grammar_str, @@ -177,10 +188,6 @@ extern "C" auto llama_rs_memory_seq_pos_max( if (mem == nullptr) { return LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM; } - uint32_t const n_seq_max = llama_n_seq_max(ctx); - if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max) { - return LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE; - } *out_position = llama_memory_seq_pos_max(mem, seq_id); return LLAMA_RS_MEMORY_SEQ_POS_MAX_OK; @@ -193,51 +200,6 @@ extern "C" auto llama_rs_memory_seq_pos_max( } } -extern "C" auto llama_rs_encode( - struct llama_context * ctx, - struct llama_batch batch, - int32_t * out_llama_cpp_return_code, - char ** out_error) -> llama_rs_encode_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - if (out_llama_cpp_return_code != nullptr) { - *out_llama_cpp_return_code = 0; - } - if (ctx == nullptr) { - return LLAMA_RS_ENCODE_NULL_CTX_ARG; - } - try { - const auto * model = llama_get_model(ctx); - if (model == nullptr) { - return LLAMA_RS_ENCODE_NULL_MODEL; - } - if (!llama_model_has_encoder(model)) { - return LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER; - } - int32_t const result = llama_encode(ctx, batch); - if (result != 0) { - if (out_llama_cpp_return_code != nullptr) { - *out_llama_cpp_return_code = result; - } - if (result == -2) { - return LLAMA_RS_ENCODE_OUT_OF_MEMORY; - } - if (result == -3) { - return LLAMA_RS_ENCODE_COMPUTE_FAILED; - } - return LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE; - } - return LLAMA_RS_ENCODE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_ENCODE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_ENCODE_ERROR_STRING_ALLOCATION_FAILED); - } -} - extern "C" auto llama_rs_memory_seq_add( const struct llama_context * ctx, llama_seq_id seq_id, @@ -252,14 +214,6 @@ extern "C" auto llama_rs_memory_seq_add( return LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG; } try { - const auto * model = llama_get_model(ctx); - if (model == nullptr) { - return LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL; - } - const auto rope = llama_model_rope_type(model); - if (rope == LLAMA_ROPE_TYPE_MROPE || rope == LLAMA_ROPE_TYPE_VISION || rope == LLAMA_ROPE_TYPE_IMROPE) { - return LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE; - } auto * mem = llama_get_memory(ctx); if (mem == nullptr) { return LLAMA_RS_MEMORY_SEQ_ADD_NULL_MEM; @@ -289,14 +243,6 @@ extern "C" auto llama_rs_memory_seq_div( return LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG; } try { - const auto * model = llama_get_model(ctx); - if (model == nullptr) { - return LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL; - } - const auto rope = llama_model_rope_type(model); - if (rope == LLAMA_ROPE_TYPE_MROPE || rope == LLAMA_ROPE_TYPE_VISION || rope == LLAMA_ROPE_TYPE_IMROPE) { - return LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE; - } auto * mem = llama_get_memory(ctx); if (mem == nullptr) { return LLAMA_RS_MEMORY_SEQ_DIV_NULL_MEM; @@ -359,6 +305,7 @@ extern "C" auto llama_rs_sampler_accept( return LLAMA_RS_SAMPLER_ACCEPT_NULL_OUT_ERROR_ARG; } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_ACCEPT_THROWS); llama_sampler_accept(sampler, token); return LLAMA_RS_SAMPLER_ACCEPT_OK; } catch (...) { @@ -440,44 +387,39 @@ extern "C" auto llama_rs_new_context_with_model( } } -extern "C" auto llama_rs_decode( +extern "C" auto llama_rs_process( struct llama_context * ctx, - struct llama_batch batch, + enum llama_process_type process_type, + struct llama_batch_ext * batch, int32_t * out_llama_cpp_return_code, - char ** out_error) -> llama_rs_decode_status { - if (out_error != nullptr) { - *out_error = nullptr; + char ** out_error) -> llama_rs_process_status { + if (out_error == nullptr) { + return LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG; } - if (out_llama_cpp_return_code != nullptr) { - *out_llama_cpp_return_code = 0; + *out_error = nullptr; + if (out_llama_cpp_return_code == nullptr) { + return LLAMA_RS_PROCESS_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG; } + *out_llama_cpp_return_code = 0; if (ctx == nullptr) { - return LLAMA_RS_DECODE_NULL_CTX_ARG; + return LLAMA_RS_PROCESS_NULL_CTX_ARG; } - if (out_error == nullptr) { - return LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG; + if (batch == nullptr) { + return LLAMA_RS_PROCESS_NULL_BATCH_ARG; } try { - int32_t const result = llama_decode(ctx, batch); + int32_t const result = llama_process(ctx, process_type, batch); if (result != 0) { - if (out_llama_cpp_return_code != nullptr) { - *out_llama_cpp_return_code = result; - } - if (result == -2) { - return LLAMA_RS_DECODE_OUT_OF_MEMORY; - } - if (result == -3) { - return LLAMA_RS_DECODE_COMPUTE_FAILED; - } - return LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE; + *out_llama_cpp_return_code = result; + return LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE; } - return LLAMA_RS_DECODE_OK; + return LLAMA_RS_PROCESS_OK; } catch (...) { return llama_rs_report_current_exception( out_error, - LLAMA_RS_DECODE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_DECODE_ERROR_STRING_ALLOCATION_FAILED); + LLAMA_RS_PROCESS_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_PROCESS_ERROR_STRING_ALLOCATION_FAILED); } } @@ -540,6 +482,7 @@ extern "C" auto llama_rs_sampler_apply( return LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG; } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_APPLY_THROWS); llama_sampler_apply(sampler, data_array); return LLAMA_RS_SAMPLER_APPLY_OK; } catch (...) { @@ -550,3 +493,34 @@ extern "C" auto llama_rs_sampler_apply( LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED); } } + +extern "C" auto llama_rs_token_to_piece( + const struct llama_vocab * vocab, + llama_token token, + char * buf, + int32_t length, + int32_t lstrip, + bool special) -> int32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_TOKEN_TO_PIECE_FAILS) || + (special && llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_SPECIAL_TOKEN_TO_PIECE_FAILS))) { + return -(length + 1); + } + + return llama_token_to_piece(vocab, token, buf, length, lstrip, special); +} + +extern "C" auto llama_rs_backend_device_text(const char * text) -> const char * { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TEXT_NULL)) { + return nullptr; + } + + return text; +} + +extern "C" auto llama_rs_backend_device_type(ggml_backend_dev_t dev) -> uint32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TYPE_UNKNOWN)) { + return GGML_BACKEND_DEVICE_TYPE_META + 1; + } + + return ggml_backend_dev_type(dev); +} diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index b9a7330d1..857a3d313 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -1,6 +1,7 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include +#include #include #include @@ -24,12 +25,12 @@ typedef enum llama_rs_json_schema_to_grammar_status { LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_LLAMA_CPP_OUT_OF_MEMORY, LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_SCHEMA, + LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_JSON, LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION, } llama_rs_json_schema_to_grammar_status; llama_rs_json_schema_to_grammar_status llama_rs_json_schema_to_grammar( const char * schema_json, - bool force_gbnf, char ** out_grammar, char ** out_error); @@ -114,7 +115,6 @@ typedef enum llama_rs_memory_seq_pos_max_status { LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_POSITION_ARG, LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_ERROR_ARG, LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM, - LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE, LLAMA_RS_MEMORY_SEQ_POS_MAX_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_OUT_OF_MEMORY, LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, @@ -126,30 +126,9 @@ llama_rs_memory_seq_pos_max_status llama_rs_memory_seq_pos_max( llama_pos * out_position, char ** out_error); -typedef enum llama_rs_encode_status { - LLAMA_RS_ENCODE_OK = 0, - LLAMA_RS_ENCODE_NULL_CTX_ARG, - LLAMA_RS_ENCODE_NULL_MODEL, - LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER, - LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - LLAMA_RS_ENCODE_OUT_OF_MEMORY, - LLAMA_RS_ENCODE_COMPUTE_FAILED, - LLAMA_RS_ENCODE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_ENCODE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_encode_status; - -llama_rs_encode_status llama_rs_encode( - struct llama_context * ctx, - struct llama_batch batch, - int32_t * out_llama_cpp_return_code, - char ** out_error); - typedef enum llama_rs_memory_seq_add_status { LLAMA_RS_MEMORY_SEQ_ADD_OK = 0, LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG, - LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL, - LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE, LLAMA_RS_MEMORY_SEQ_ADD_NULL_MEM, LLAMA_RS_MEMORY_SEQ_ADD_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_OUT_OF_MEMORY, @@ -167,8 +146,6 @@ llama_rs_memory_seq_add_status llama_rs_memory_seq_add( typedef enum llama_rs_memory_seq_div_status { LLAMA_RS_MEMORY_SEQ_DIV_OK = 0, LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG, - LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL, - LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE, LLAMA_RS_MEMORY_SEQ_DIV_NULL_MEM, LLAMA_RS_MEMORY_SEQ_DIV_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_OUT_OF_MEMORY, @@ -217,21 +194,22 @@ llama_rs_new_context_with_model_status llama_rs_new_context_with_model( struct llama_context ** out_ctx, char ** out_error); -typedef enum llama_rs_decode_status { - LLAMA_RS_DECODE_OK = 0, - LLAMA_RS_DECODE_NULL_CTX_ARG, - LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG, - LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - LLAMA_RS_DECODE_OUT_OF_MEMORY, - LLAMA_RS_DECODE_COMPUTE_FAILED, - LLAMA_RS_DECODE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_DECODE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_decode_status; - -llama_rs_decode_status llama_rs_decode( +typedef enum llama_rs_process_status { + LLAMA_RS_PROCESS_OK = 0, + LLAMA_RS_PROCESS_NULL_CTX_ARG, + LLAMA_RS_PROCESS_NULL_BATCH_ARG, + LLAMA_RS_PROCESS_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG, + LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + LLAMA_RS_PROCESS_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_PROCESS_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_process_status; + +llama_rs_process_status llama_rs_process( struct llama_context * ctx, - struct llama_batch batch, + enum llama_process_type process_type, + struct llama_batch_ext * batch, int32_t * out_llama_cpp_return_code, char ** out_error); @@ -272,6 +250,18 @@ llama_rs_sampler_apply_status llama_rs_sampler_apply( struct llama_token_data_array * data_array, char ** out_error); +int32_t llama_rs_token_to_piece( + const struct llama_vocab * vocab, + llama_token token, + char * buf, + int32_t length, + int32_t lstrip, + bool special); + +const char * llama_rs_backend_device_text(const char * text); + +uint32_t llama_rs_backend_device_type(ggml_backend_dev_t dev); + #ifdef __cplusplus } #endif diff --git a/llama-cpp-bindings-sys/wrapper_context.cpp b/llama-cpp-bindings-sys/wrapper_context.cpp index 4b06b775a..e97589653 100644 --- a/llama-cpp-bindings-sys/wrapper_context.cpp +++ b/llama-cpp-bindings-sys/wrapper_context.cpp @@ -1,87 +1,52 @@ #include "wrapper_context.h" +#include "wrapper_fault_injection.h" +#include #include -#include "llama.cpp/include/llama.h" -#include "llama.cpp/src/llama-context.h" -#include "llama.cpp/src/llama-ext.h" -#include "llama.cpp/src/llama-memory.h" -#include "llama.cpp/src/llama-model.h" -#include "wrapper_report_current_exception.h" +#include +#include -extern "C" auto llama_rs_context_decodes_batches_in_one_micro_batch(const struct llama_context * ctx) -> bool { - return llama_get_memory(ctx) == nullptr || !ctx->get_cparams().causal_attn; +extern "C" auto llama_rs_model_causal_attn(const struct llama_model * model) -> bool { + return model->hparams.causal_attn; } -extern "C" auto llama_rs_context_embeddings_enabled(const struct llama_context * ctx) -> bool { - return ctx->get_cparams().embeddings; -} +extern "C" auto llama_rs_pooling_type(const struct llama_context * ctx) -> int32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_POOLING_TYPE_UNKNOWN)) { + return LLAMA_POOLING_TYPE_RANK + 1; + } -extern "C" auto llama_rs_context_embeddings_nextn_enabled(const struct llama_context * ctx) -> bool { - return ctx->get_cparams().embeddings_nextn; + return llama_pooling_type(ctx); } -extern "C" auto llama_rs_context_embeddings_nextn_masked(const struct llama_context * ctx) -> bool { - return ctx->get_cparams().embeddings_nextn_masked; -} +extern "C" auto llama_rs_model_rope_type(const struct llama_model * model) -> int32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_ROPE_TYPE_UNKNOWN)) { + return LLAMA_ROPE_TYPE_IMROPE + 1; + } -extern "C" void llama_rs_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked) { - llama_set_embeddings_nextn(ctx, value, masked); + return llama_model_rope_type(model); } -extern "C" auto llama_rs_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t token_index) -> float * { - return llama_get_embeddings_nextn_ith(ctx, token_index); +extern "C" auto llama_rs_context_produced_logits(struct llama_context * ctx) -> bool { + return llama_get_logits(ctx) != nullptr; } -extern "C" auto llama_rs_context_graph_produces_nextn_hidden_state( - struct llama_context * ctx, - bool * out_produces_nextn_hidden_state, - char ** out_error) -> llama_rs_context_nextn_hidden_state_probe_status { - if (out_produces_nextn_hidden_state != nullptr) { - *out_produces_nextn_hidden_state = false; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (ctx == nullptr) { - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG; +extern "C" auto llama_rs_get_logits_ith(struct llama_context * ctx, int32_t token_index) -> float * { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_GET_LOGITS_ITH_RETURNS_NULL)) { + return nullptr; } - if (out_produces_nextn_hidden_state == nullptr) { - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG; - } - try { - constexpr uint32_t smallest_ubatch_tokens = 1; - constexpr uint32_t smallest_ubatch_sequences = 1; - constexpr uint32_t smallest_ubatch_outputs = 1; - constexpr bool split_graph_without_allocating_buffers = true; - llama_memory_context_ptr memory_context; - llama_memory_t memory = llama_get_memory(ctx); - if (memory != nullptr) { - memory_context = memory->init_full(); - } - if (ctx->graph_reserve( - smallest_ubatch_tokens, - smallest_ubatch_sequences, - smallest_ubatch_outputs, - memory_context.get(), - split_graph_without_allocating_buffers) == nullptr) { - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_FAILED_TO_RESERVE_GRAPH; - } - *out_produces_nextn_hidden_state = ctx->get_gf_res_reserve()->get_h_nextn() != nullptr; - return LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_ERROR_STRING_ALLOCATION_FAILED); - } + return llama_get_logits_ith(ctx, token_index); } -extern "C" auto llama_rs_model_causal_attn(const struct llama_model * model) -> bool { - return model->hparams.causal_attn; +extern "C" auto llama_rs_set_adapters_lora( + struct llama_context * ctx, + struct llama_adapter_lora ** adapters, + size_t n_adapters, + float * scales) -> int32_t { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_SET_ADAPTERS_LORA_FAILS)) { + return -1; + } + + return llama_set_adapters_lora(ctx, adapters, n_adapters, scales); } diff --git a/llama-cpp-bindings-sys/wrapper_context.h b/llama-cpp-bindings-sys/wrapper_context.h index c8d58b28c..0f944df77 100644 --- a/llama-cpp-bindings-sys/wrapper_context.h +++ b/llama-cpp-bindings-sys/wrapper_context.h @@ -1,44 +1,32 @@ #pragma once #include +#include #include #ifdef __cplusplus extern "C" { #endif +struct llama_adapter_lora; struct llama_context; struct llama_model; -typedef enum llama_rs_context_nextn_hidden_state_probe_status { - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK = 0, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_FAILED_TO_RESERVE_GRAPH, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_context_nextn_hidden_state_probe_status; - -bool llama_rs_context_decodes_batches_in_one_micro_batch(const struct llama_context * ctx); - -bool llama_rs_context_embeddings_enabled(const struct llama_context * ctx); +bool llama_rs_model_causal_attn(const struct llama_model * model); -bool llama_rs_context_embeddings_nextn_enabled(const struct llama_context * ctx); +int32_t llama_rs_pooling_type(const struct llama_context * ctx); -bool llama_rs_context_embeddings_nextn_masked(const struct llama_context * ctx); +int32_t llama_rs_model_rope_type(const struct llama_model * model); -void llama_rs_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); +bool llama_rs_context_produced_logits(struct llama_context * ctx); -float * llama_rs_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t token_index); +float * llama_rs_get_logits_ith(struct llama_context * ctx, int32_t token_index); -llama_rs_context_nextn_hidden_state_probe_status llama_rs_context_graph_produces_nextn_hidden_state( +int32_t llama_rs_set_adapters_lora( struct llama_context * ctx, - bool * out_produces_nextn_hidden_state, - char ** out_error); - -bool llama_rs_model_causal_attn(const struct llama_model * model); + struct llama_adapter_lora ** adapters, + size_t n_adapters, + float * scales); #ifdef __cplusplus } diff --git a/llama-cpp-bindings-sys/wrapper_fault_injection.cpp b/llama-cpp-bindings-sys/wrapper_fault_injection.cpp new file mode 100644 index 000000000..184a9ba7a --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_fault_injection.cpp @@ -0,0 +1,82 @@ +#include "wrapper_fault_injection.h" +#include "wrapper_utils.h" + +#include +#include +#include + +#ifdef LLAMA_RS_FAULT_INJECTION +namespace { + +constexpr const char * INJECTED_FAULT_MESSAGE = "llama-cpp-bindings injected a fault"; + +struct armed_fault { + llama_rs_fault_point point = LLAMA_RS_FAULT_POINT_NONE; + size_t passing_hits = 0; +}; + +auto armed_fault_on_this_thread() -> armed_fault & { + thread_local armed_fault fault; + + return fault; +} + +} // namespace + +extern "C" void llama_rs_fault_injection_arm(llama_rs_fault_point point, size_t passing_hits) { + armed_fault_on_this_thread() = armed_fault{point, passing_hits}; +} + +extern "C" void llama_rs_fault_injection_disarm() { + armed_fault_on_this_thread() = armed_fault{}; +} + +extern "C" auto llama_rs_fault_injection_message() -> const char * { + return INJECTED_FAULT_MESSAGE; +} + +auto llama_rs_fault_injected(llama_rs_fault_point point) -> bool { + armed_fault & fault = armed_fault_on_this_thread(); + + if (fault.point != point) { + return false; + } + + if (fault.passing_hits > 0) { + fault.passing_hits -= 1; + + return false; + } + + return true; +} + +void llama_rs_throw_if_fault_injected(llama_rs_fault_point point) { + if (llama_rs_fault_injected(point)) { + throw std::runtime_error(INJECTED_FAULT_MESSAGE); + } +} +#else +auto llama_rs_fault_injected([[maybe_unused]] llama_rs_fault_point point) -> bool { + return false; +} + +void llama_rs_throw_if_fault_injected([[maybe_unused]] llama_rs_fault_point point) {} +#endif + +auto llama_rs_text_view_unless_fault_injected( + llama_rs_fault_point point, + const std::string & text, + size_t * out_length) -> const char * { + if (llama_rs_fault_injected(point)) { + static const std::string not_utf8 = "\xff"; + + return llama_rs_string_view(not_utf8, out_length); + } + + return llama_rs_string_view(text, out_length); +} + +extern "C" auto llama_rs_fault_point_reached(llama_rs_fault_point point) -> bool { + return llama_rs_fault_injected(point); +} diff --git a/llama-cpp-bindings-sys/wrapper_fault_injection.h b/llama-cpp-bindings-sys/wrapper_fault_injection.h new file mode 100644 index 000000000..e8c07424b --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_fault_injection.h @@ -0,0 +1,74 @@ +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum llama_rs_fault_point { + LLAMA_RS_FAULT_POINT_NONE = 0, + LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TEXT_NULL, + LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TYPE_UNKNOWN, + LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS, + LLAMA_RS_FAULT_POINT_BATCH_EXT_INIT_THROWS, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_VALUE_NOT_UTF8, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS, + LLAMA_RS_FAULT_POINT_CHAT_SESSION_PROMPT_PART_TEXT_NOT_UTF8, + LLAMA_RS_FAULT_POINT_GET_LOGITS_ITH_RETURNS_NULL, + LLAMA_RS_FAULT_POINT_LLGUIDANCE_FACTORY_FAILS, + LLAMA_RS_FAULT_POINT_MTMD_EVAL_CHUNK_SINGLE_THROWS, + LLAMA_RS_FAULT_POINT_MTMD_GET_MEMORY_USAGE_THROWS, + LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNK_TYPE_UNKNOWN, + LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL, + LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_INIT_THROWS, + LLAMA_RS_FAULT_POINT_MTMD_TOKENIZE_FROM_PARTS_THROWS, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_CONTENT_NOT_UTF8, + LLAMA_RS_FAULT_POINT_POOLING_TYPE_UNKNOWN, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_REASONING_CONTENT_NOT_UTF8, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NOT_UTF8, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ID_NOT_UTF8, + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_NAME_NOT_UTF8, + LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS, + LLAMA_RS_FAULT_POINT_ROPE_TYPE_UNKNOWN, + LLAMA_RS_FAULT_POINT_SAMPLER_ACCEPT_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_APPLY_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_RETURNS_NULL, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_THROWS, + LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS, + LLAMA_RS_FAULT_POINT_SET_ADAPTERS_LORA_FAILS, + LLAMA_RS_FAULT_POINT_SPECIAL_TOKEN_TO_PIECE_FAILS, + LLAMA_RS_FAULT_POINT_TOKEN_TO_PIECE_FAILS, +} llama_rs_fault_point; + +bool llama_rs_fault_point_reached(llama_rs_fault_point point); + +#ifdef LLAMA_RS_FAULT_INJECTION +void llama_rs_fault_injection_arm(llama_rs_fault_point point, size_t passing_hits); + +void llama_rs_fault_injection_disarm(void); + +const char * llama_rs_fault_injection_message(void); +#endif + +#ifdef __cplusplus +} + +#include +#include + +auto llama_rs_fault_injected(llama_rs_fault_point point) -> bool; + +void llama_rs_throw_if_fault_injected(llama_rs_fault_point point); + +auto llama_rs_text_view_unless_fault_injected( + llama_rs_fault_point point, + const std::string & text, + size_t * out_length) -> const char *; +#endif diff --git a/llama-cpp-bindings-sys/wrapper_fit.cpp b/llama-cpp-bindings-sys/wrapper_fit.cpp index 7cb39e651..66b27798b 100644 --- a/llama-cpp-bindings-sys/wrapper_fit.cpp +++ b/llama-cpp-bindings-sys/wrapper_fit.cpp @@ -1,12 +1,12 @@ #include "wrapper_fit.h" -#include "llama.h" -#include "ggml.h" +#include +#include #include "wrapper_report_current_exception.h" #include #include -#include "llama.cpp/common/fit.h" +#include extern "C" auto llama_rs_fit_params( const char * path_model, diff --git a/llama-cpp-bindings-sys/wrapper_fit.h b/llama-cpp-bindings-sys/wrapper_fit.h index f9485cde6..514edba17 100644 --- a/llama-cpp-bindings-sys/wrapper_fit.h +++ b/llama-cpp-bindings-sys/wrapper_fit.h @@ -1,7 +1,7 @@ #pragma once -#include "llama.cpp/ggml/include/ggml.h" -#include "llama.cpp/include/llama.h" +#include +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_gbnf.cpp b/llama-cpp-bindings-sys/wrapper_gbnf.cpp index 2cff7b500..644f497af 100644 --- a/llama-cpp-bindings-sys/wrapper_gbnf.cpp +++ b/llama-cpp-bindings-sys/wrapper_gbnf.cpp @@ -1,7 +1,7 @@ #include "wrapper_gbnf.h" #include "wrapper_report_current_exception.h" -#include "llama.cpp/src/llama-grammar.h" +#include extern "C" auto llama_rs_validate_gbnf( @@ -40,7 +40,7 @@ extern "C" auto llama_rs_validate_gbnf( vocab, grammar_str, grammar_root, false, nullptr, 0, nullptr, 0); if (grammar == nullptr) { - return LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION; + return LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR; } llama_grammar_free_impl(grammar); diff --git a/llama-cpp-bindings-sys/wrapper_gbnf.h b/llama-cpp-bindings-sys/wrapper_gbnf.h index 337167524..8ca391ef9 100644 --- a/llama-cpp-bindings-sys/wrapper_gbnf.h +++ b/llama-cpp-bindings-sys/wrapper_gbnf.h @@ -11,7 +11,7 @@ typedef enum llama_rs_gbnf_validation_status { LLAMA_RS_GBNF_VALIDATION_SYNTAX_ERROR, LLAMA_RS_GBNF_VALIDATION_EMPTY_RULE_SET, LLAMA_RS_GBNF_VALIDATION_ROOT_SYMBOL_MISSING, - LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION, + LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR, LLAMA_RS_GBNF_VALIDATION_NULL_GRAMMAR_ARG, LLAMA_RS_GBNF_VALIDATION_NULL_ROOT_ARG, LLAMA_RS_GBNF_VALIDATION_NULL_OUT_ERROR_ARG, diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.cpp b/llama-cpp-bindings-sys/wrapper_mtmd.cpp index b56a0fb78..79637a3a2 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.cpp +++ b/llama-cpp-bindings-sys/wrapper_mtmd.cpp @@ -1,7 +1,8 @@ #include "wrapper_mtmd.h" -#include "llama.h" -#include "tools/mtmd/mtmd.h" -#include "tools/mtmd/mtmd-helper.h" +#include "wrapper_fault_injection.h" +#include +#include +#include #include "wrapper_report_current_exception.h" #include @@ -202,6 +203,8 @@ extern "C" auto llama_rs_mtmd_tokenize_from_parts( } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_TOKENIZE_FROM_PARTS_THROWS); + int32_t const result = mtmd_tokenize_from_parts(ctx, output, parts, num_parts, add_special); switch (result) { case 0: @@ -292,6 +295,8 @@ extern "C" auto llama_rs_mtmd_eval_chunk_single( } try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_EVAL_CHUNK_SINGLE_THROWS); + int32_t const result = mtmd_helper_eval_chunk_single( ctx, lctx, chunk, n_past, seq_id, n_batch, logits_last, out_new_n_past); if (result != 0) { @@ -309,3 +314,194 @@ extern "C" auto llama_rs_mtmd_eval_chunk_single( LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED); } } + +extern "C" auto llama_rs_mtmd_get_memory_usage( + const char * mmproj_path, + struct mtmd_context_params ctx_params, + llama_rs_mtmd_device_memory * out_device_memory, + size_t device_memory_capacity, + size_t * out_device_memory_count, + int32_t * out_image_max_tokens, + bool * out_use_non_causal, + char ** out_error) -> llama_rs_mtmd_get_memory_usage_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (mmproj_path == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_MMPROJ_PATH_ARG; + } + if (out_device_memory == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_ARG; + } + if (out_device_memory_count == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_COUNT_ARG; + } + if (out_image_max_tokens == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_IMAGE_MAX_TOKENS_ARG; + } + if (out_use_non_causal == nullptr) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_USE_NON_CAUSAL_ARG; + } + *out_device_memory_count = 0; + + try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_GET_MEMORY_USAGE_THROWS); + + const mtmd_memory_usage usage = mtmd_get_memory_usage(mmproj_path, ctx_params); + + if (usage.backend_mem_usage.size() > device_memory_capacity) { + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_DEVICE_MEMORY_CAPACITY_EXCEEDED; + } + + const gsl::span device_memory(out_device_memory, device_memory_capacity); + size_t device_memory_count = 0; + + for (const auto & [device, bytes] : usage.backend_mem_usage) { + device_memory[device_memory_count] = llama_rs_mtmd_device_memory{device, bytes}; + device_memory_count += 1; + } + + *out_device_memory_count = device_memory_count; + *out_image_max_tokens = usage.image_max_tokens; + *out_use_non_causal = usage.use_non_causal; + + return LLAMA_RS_MTMD_GET_MEMORY_USAGE_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_mtmd_bitmap_init( + uint32_t width, + uint32_t height, + const unsigned char * data, + struct mtmd_bitmap ** out_bitmap, + char ** out_error) -> llama_rs_mtmd_bitmap_init_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (out_bitmap == nullptr) { + return LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG; + } + *out_bitmap = nullptr; + if (data == nullptr) { + return LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG; + } + + try { + *out_bitmap = mtmd_bitmap_init(width, height, data); + + return LLAMA_RS_MTMD_BITMAP_INIT_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_BITMAP_INIT_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_mtmd_bitmap_init_from_audio( + size_t n_samples, + const float * data, + struct mtmd_bitmap ** out_bitmap, + char ** out_error) -> llama_rs_mtmd_bitmap_init_from_audio_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (out_bitmap == nullptr) { + return LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG; + } + *out_bitmap = nullptr; + if (data == nullptr) { + return LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG; + } + + try { + *out_bitmap = mtmd_bitmap_init_from_audio(n_samples, data); + + return LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_mtmd_input_chunks_init( + struct mtmd_input_chunks ** out_chunks, + char ** out_error) -> llama_rs_mtmd_input_chunks_init_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (out_chunks == nullptr) { + return LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG; + } + *out_chunks = nullptr; + + try { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_INIT_THROWS); + + *out_chunks = mtmd_input_chunks_init(); + + return LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_mtmd_input_chunk_copy( + const struct mtmd_input_chunk * chunk, + struct mtmd_input_chunk ** out_chunk, + char ** out_error) -> llama_rs_mtmd_input_chunk_copy_status { + if (out_error != nullptr) { + *out_error = nullptr; + } + if (out_chunk == nullptr) { + return LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG; + } + *out_chunk = nullptr; + if (chunk == nullptr) { + return LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG; + } + + try { + *out_chunk = mtmd_input_chunk_copy(chunk); + + return LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK; + } catch (...) { + return llama_rs_report_current_exception( + out_error, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_ERROR_STRING_ALLOCATION_FAILED); + } +} + +extern "C" auto llama_rs_mtmd_input_chunks_get(const struct mtmd_input_chunks * chunks, size_t index) + -> const struct mtmd_input_chunk * { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL)) { + return nullptr; + } + + return mtmd_input_chunks_get(chunks, index); +} + +extern "C" auto llama_rs_mtmd_input_chunk_get_type(const struct mtmd_input_chunk * chunk) + -> mtmd_input_chunk_type { + if (llama_rs_fault_injected(LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNK_TYPE_UNKNOWN)) { + return MTMD_INPUT_CHUNK_TYPE_COUNT; + } + + return mtmd_input_chunk_get_type(chunk); +} diff --git a/llama-cpp-bindings-sys/wrapper_mtmd.h b/llama-cpp-bindings-sys/wrapper_mtmd.h index 192d7a237..b6804c880 100644 --- a/llama-cpp-bindings-sys/wrapper_mtmd.h +++ b/llama-cpp-bindings-sys/wrapper_mtmd.h @@ -1,8 +1,8 @@ #pragma once -#include "llama.cpp/include/llama.h" -#include "llama.cpp/tools/mtmd/mtmd.h" -#include "llama.cpp/tools/mtmd/mtmd-helper.h" +#include +#include +#include #include #include @@ -150,6 +150,97 @@ llama_rs_mtmd_eval_chunk_single_status llama_rs_mtmd_eval_chunk_single( int32_t * out_llama_cpp_return_code, char ** out_error); +typedef struct llama_rs_mtmd_device_memory { + ggml_backend_dev_t device; + size_t bytes; +} llama_rs_mtmd_device_memory; + +typedef enum llama_rs_mtmd_get_memory_usage_status { + LLAMA_RS_MTMD_GET_MEMORY_USAGE_OK = 0, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_MMPROJ_PATH_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_COUNT_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_IMAGE_MAX_TOKENS_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_USE_NON_CAUSAL_ARG, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_DEVICE_MEMORY_CAPACITY_EXCEEDED, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_get_memory_usage_status; + +llama_rs_mtmd_get_memory_usage_status llama_rs_mtmd_get_memory_usage( + const char * mmproj_path, + struct mtmd_context_params ctx_params, + llama_rs_mtmd_device_memory * out_device_memory, + size_t device_memory_capacity, + size_t * out_device_memory_count, + int32_t * out_image_max_tokens, + bool * out_use_non_causal, + char ** out_error); + +typedef enum llama_rs_mtmd_bitmap_init_status { + LLAMA_RS_MTMD_BITMAP_INIT_OK = 0, + LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG, + LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG, + LLAMA_RS_MTMD_BITMAP_INIT_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_bitmap_init_status; + +llama_rs_mtmd_bitmap_init_status llama_rs_mtmd_bitmap_init( + uint32_t width, + uint32_t height, + const unsigned char * data, + struct mtmd_bitmap ** out_bitmap, + char ** out_error); + +typedef enum llama_rs_mtmd_bitmap_init_from_audio_status { + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK = 0, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_bitmap_init_from_audio_status; + +llama_rs_mtmd_bitmap_init_from_audio_status llama_rs_mtmd_bitmap_init_from_audio( + size_t n_samples, + const float * data, + struct mtmd_bitmap ** out_bitmap, + char ** out_error); + +typedef enum llama_rs_mtmd_input_chunks_init_status { + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK = 0, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_input_chunks_init_status; + +llama_rs_mtmd_input_chunks_init_status llama_rs_mtmd_input_chunks_init( + struct mtmd_input_chunks ** out_chunks, + char ** out_error); + +typedef enum llama_rs_mtmd_input_chunk_copy_status { + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK = 0, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_ERROR_STRING_ALLOCATION_FAILED, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_OUT_OF_MEMORY, + LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION, +} llama_rs_mtmd_input_chunk_copy_status; + +llama_rs_mtmd_input_chunk_copy_status llama_rs_mtmd_input_chunk_copy( + const struct mtmd_input_chunk * chunk, + struct mtmd_input_chunk ** out_chunk, + char ** out_error); + +const struct mtmd_input_chunk * llama_rs_mtmd_input_chunks_get( + const struct mtmd_input_chunks * chunks, + size_t index); + +enum mtmd_input_chunk_type llama_rs_mtmd_input_chunk_get_type(const struct mtmd_input_chunk * chunk); + #ifdef __cplusplus } #endif diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp new file mode 100644 index 000000000..226768b36 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat.cpp @@ -0,0 +1,51 @@ +#include "wrapper_parsed_chat.h" +#include "wrapper_fault_injection.h" +#include "wrapper_parsed_chat_handle.h" + +#include +#include + +extern "C" auto llama_rs_parsed_chat_free(llama_rs_parsed_chat_handle handle) -> void { + std::unique_ptr(handle).reset(); +} + +extern "C" auto llama_rs_parsed_chat_content(llama_rs_parsed_chat_handle handle, size_t * out_length) + -> const char * { + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_CONTENT_NOT_UTF8, handle->message.content, out_length); +} + +extern "C" auto llama_rs_parsed_chat_reasoning_content( + llama_rs_parsed_chat_handle handle, + size_t * out_length) -> const char * { + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_REASONING_CONTENT_NOT_UTF8, handle->message.reasoning_content, out_length); +} + +extern "C" auto llama_rs_parsed_chat_tool_call_count(llama_rs_parsed_chat_handle handle) -> size_t { + return handle->message.tool_calls.size(); +} + +extern "C" auto llama_rs_parsed_chat_tool_call_id( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ID_NOT_UTF8, handle->message.tool_calls[index].id, out_length); +} + +extern "C" auto llama_rs_parsed_chat_tool_call_name( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_NAME_NOT_UTF8, handle->message.tool_calls[index].name, out_length); +} + +extern "C" auto llama_rs_parsed_chat_tool_call_arguments( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length) -> const char * { + return llama_rs_text_view_unless_fault_injected( + LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NOT_UTF8, handle->message.tool_calls[index].arguments, out_length); +} diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat.h b/llama-cpp-bindings-sys/wrapper_parsed_chat.h new file mode 100644 index 000000000..d7bc5f1bf --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat.h @@ -0,0 +1,41 @@ +#pragma once + +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct llama_rs_parsed_chat; +typedef struct llama_rs_parsed_chat * llama_rs_parsed_chat_handle; + +void llama_rs_parsed_chat_free(llama_rs_parsed_chat_handle handle); + +const char * llama_rs_parsed_chat_content(llama_rs_parsed_chat_handle handle, size_t * out_length); + +const char * llama_rs_parsed_chat_reasoning_content( + llama_rs_parsed_chat_handle handle, + size_t * out_length); + +size_t llama_rs_parsed_chat_tool_call_count(llama_rs_parsed_chat_handle handle); + +const char * llama_rs_parsed_chat_tool_call_id( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length); + +const char * llama_rs_parsed_chat_tool_call_name( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length); + +const char * llama_rs_parsed_chat_tool_call_arguments( + llama_rs_parsed_chat_handle handle, + size_t index, + size_t * out_length); + +#ifdef __cplusplus +} +#endif diff --git a/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h b/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h new file mode 100644 index 000000000..b6b318a86 --- /dev/null +++ b/llama-cpp-bindings-sys/wrapper_parsed_chat_handle.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +struct llama_rs_parsed_chat { + common_chat_msg message; +}; diff --git a/llama-cpp-bindings-sys/wrapper_reasoning.cpp b/llama-cpp-bindings-sys/wrapper_reasoning.cpp deleted file mode 100644 index 327b673e5..000000000 --- a/llama-cpp-bindings-sys/wrapper_reasoning.cpp +++ /dev/null @@ -1,160 +0,0 @@ -#include "wrapper_reasoning.h" -#include "wrapper_token_text.h" - -#include "llama.cpp/common/chat-auto-parser.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" -#include "wrapper_report_current_exception.h" - -#include -#include -#include -#include -#include - -using wrapper_helpers::token_text_or_empty; - -struct llama_rs_reasoning_markers { - std::string open; - std::vector closes; -}; - -namespace { - -auto find_reasoning_markers( - const common_chat_template & tmpl, - const char * tmpl_src, - std::string * out_start, - std::vector * out_ends) -> bool { - autoparser::generation_params probe_params; - probe_params.add_generation_prompt = true; - probe_params.enable_thinking = true; - probe_params.is_inference = false; - probe_params.add_inference = false; - probe_params.mark_input = false; - probe_params.messages = common_json::array({ - common_json{ { "role", "user" }, { "content", "ping" } }, - }); - - const std::string tmpl_src_str = tmpl_src; - if (auto specialized = common_chat_try_specialized_template(tmpl, tmpl_src_str, probe_params)) { - if (specialized->supports_thinking - && !specialized->thinking_start_tag.empty() - && !specialized->thinking_end_tags.empty()) { - *out_start = std::move(specialized->thinking_start_tag); - *out_ends = std::move(specialized->thinking_end_tags); - return true; - } - } - - autoparser::autoparser parser; - parser.analyze_template(tmpl); - if (parser.reasoning.mode != autoparser::reasoning_mode::NONE - && !parser.reasoning.start.empty() - && !parser.reasoning.end.empty()) { - *out_start = std::move(parser.reasoning.start); - out_ends->push_back(std::move(parser.reasoning.end)); - return true; - } - - return false; -} - -} // namespace - -extern "C" auto llama_rs_detect_reasoning_markers( - const struct llama_model * model, - llama_rs_reasoning_markers ** out_markers, - char ** out_error) -> llama_rs_detect_reasoning_markers_status { - if (out_markers != nullptr) { - *out_markers = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_MODEL_ARG; - } - if (out_markers == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_MARKERS_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG; - } - - try { - const char * tmpl_src = llama_model_chat_template(model, nullptr); - if (tmpl_src == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_CHAT_TEMPLATE; - } - - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(tmpl_src, bos_token, eos_token); - - auto detected = std::make_unique(); - if (!find_reasoning_markers(tmpl, tmpl_src, &detected->open, &detected->closes)) { - return LLAMA_RS_DETECT_REASONING_MARKERS_OK; - } - - *out_markers = detected.release(); - - return LLAMA_RS_DETECT_REASONING_MARKERS_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_reasoning_markers_open( - const llama_rs_reasoning_markers * markers) -> const char * { - if (markers == nullptr) { - return nullptr; - } - return markers->open.c_str(); -} - -extern "C" auto llama_rs_reasoning_markers_close_count( - const llama_rs_reasoning_markers * markers) -> size_t { - if (markers == nullptr) { - return 0; - } - return markers->closes.size(); -} - -extern "C" auto llama_rs_reasoning_markers_close_at( - const llama_rs_reasoning_markers * markers, - size_t index) -> const char * { - if (markers == nullptr || index >= markers->closes.size()) { - return nullptr; - } - return markers->closes[index].c_str(); -} - -extern "C" auto llama_rs_reasoning_markers_free( - llama_rs_reasoning_markers * markers, - char ** out_error) -> llama_rs_reasoning_markers_free_status { - if (out_error != nullptr) { - *out_error = nullptr; - } - try { - const std::unique_ptr reclaimed(markers); - return LLAMA_RS_REASONING_MARKERS_FREE_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_REASONING_MARKERS_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_REASONING_MARKERS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - LLAMA_RS_REASONING_MARKERS_FREE_ERROR_STRING_ALLOCATION_FAILED); - } -} diff --git a/llama-cpp-bindings-sys/wrapper_reasoning.h b/llama-cpp-bindings-sys/wrapper_reasoning.h deleted file mode 100644 index 9370c4323..000000000 --- a/llama-cpp-bindings-sys/wrapper_reasoning.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" -#include "wrapper_utils.h" - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum llama_rs_detect_reasoning_markers_status { - LLAMA_RS_DETECT_REASONING_MARKERS_OK = 0, - LLAMA_RS_DETECT_REASONING_MARKERS_NULL_MODEL_ARG, - LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_MARKERS_ARG, - LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG, - LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_CHAT_TEMPLATE, - LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_VOCAB, - LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_detect_reasoning_markers_status; - -typedef struct llama_rs_reasoning_markers llama_rs_reasoning_markers; - -llama_rs_detect_reasoning_markers_status llama_rs_detect_reasoning_markers( - const struct llama_model * model, - llama_rs_reasoning_markers ** out_markers, - char ** out_error); - -const char * llama_rs_reasoning_markers_open(const llama_rs_reasoning_markers * markers); -size_t llama_rs_reasoning_markers_close_count(const llama_rs_reasoning_markers * markers); -const char * llama_rs_reasoning_markers_close_at( - const llama_rs_reasoning_markers * markers, - size_t index); -typedef enum llama_rs_reasoning_markers_free_status { - LLAMA_RS_REASONING_MARKERS_FREE_OK = 0, - LLAMA_RS_REASONING_MARKERS_FREE_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_REASONING_MARKERS_FREE_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_REASONING_MARKERS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, -} llama_rs_reasoning_markers_free_status; - -llama_rs_reasoning_markers_free_status llama_rs_reasoning_markers_free( - llama_rs_reasoning_markers * markers, - char ** out_error); - -#ifdef __cplusplus -} -#endif diff --git a/llama-cpp-bindings-sys/wrapper_sampler_init.cpp b/llama-cpp-bindings-sys/wrapper_sampler_init.cpp index ee26f71bc..8ce8e4d13 100644 --- a/llama-cpp-bindings-sys/wrapper_sampler_init.cpp +++ b/llama-cpp-bindings-sys/wrapper_sampler_init.cpp @@ -1,6 +1,7 @@ #include "wrapper_sampler_init.h" +#include "wrapper_fault_injection.h" -#include "llama.cpp/include/llama.h" +#include #include "wrapper_report_current_exception.h" #include @@ -61,14 +62,20 @@ extern "C" auto llama_rs_sampler_chain_init( extern "C" auto llama_rs_sampler_init_greedy( struct llama_sampler ** out_sampler, char ** out_error) -> llama_rs_sampler_init_status { - return init_sampler(out_sampler, out_error, []() -> struct llama_sampler * { return llama_sampler_init_greedy(); }); + return init_sampler(out_sampler, out_error, []() -> struct llama_sampler * { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS); + return llama_sampler_init_greedy(); + }); } extern "C" auto llama_rs_sampler_init_dist( uint32_t seed, struct llama_sampler ** out_sampler, char ** out_error) -> llama_rs_sampler_init_status { - return init_sampler(out_sampler, out_error, [seed]() -> struct llama_sampler * { return llama_sampler_init_dist(seed); }); + return init_sampler(out_sampler, out_error, [seed]() -> struct llama_sampler * { + llama_rs_throw_if_fault_injected(LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS); + return llama_sampler_init_dist(seed); + }); } extern "C" auto llama_rs_sampler_init_top_k( diff --git a/llama-cpp-bindings-sys/wrapper_sampler_init.h b/llama-cpp-bindings-sys/wrapper_sampler_init.h index dbf4ac229..2f7abfcb2 100644 --- a/llama-cpp-bindings-sys/wrapper_sampler_init.h +++ b/llama-cpp-bindings-sys/wrapper_sampler_init.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_state.cpp b/llama-cpp-bindings-sys/wrapper_state.cpp index 0fef5235b..230ae53ad 100644 --- a/llama-cpp-bindings-sys/wrapper_state.cpp +++ b/llama-cpp-bindings-sys/wrapper_state.cpp @@ -1,6 +1,6 @@ #include "wrapper_state.h" -#include "llama.cpp/include/llama.h" +#include #include "wrapper_report_current_exception.h" #include diff --git a/llama-cpp-bindings-sys/wrapper_state.h b/llama-cpp-bindings-sys/wrapper_state.h index 4f76fe630..199e3ccd1 100644 --- a/llama-cpp-bindings-sys/wrapper_state.h +++ b/llama-cpp-bindings-sys/wrapper_state.h @@ -1,6 +1,6 @@ #pragma once -#include "llama.cpp/include/llama.h" +#include #include #include diff --git a/llama-cpp-bindings-sys/wrapper_token_text.cpp b/llama-cpp-bindings-sys/wrapper_token_text.cpp deleted file mode 100644 index 7719e1859..000000000 --- a/llama-cpp-bindings-sys/wrapper_token_text.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "wrapper_token_text.h" -#include "llama.h" -#include - -namespace wrapper_helpers { - -auto token_text_or_empty(const llama_vocab * vocab, llama_token token) -> std::string { - if (token == LLAMA_TOKEN_NULL) { - return {}; - } - - const char * text = llama_vocab_get_text(vocab, token); - if (text == nullptr) { - return {}; - } - - return {text}; -} - -} // namespace wrapper_helpers diff --git a/llama-cpp-bindings-sys/wrapper_token_text.h b/llama-cpp-bindings-sys/wrapper_token_text.h deleted file mode 100644 index 231527e1a..000000000 --- a/llama-cpp-bindings-sys/wrapper_token_text.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" - -#include - -namespace wrapper_helpers { - -std::string token_text_or_empty(const llama_vocab * vocab, llama_token token); - -} diff --git a/llama-cpp-bindings-sys/wrapper_tool_calls.cpp b/llama-cpp-bindings-sys/wrapper_tool_calls.cpp deleted file mode 100644 index c742e50f8..000000000 --- a/llama-cpp-bindings-sys/wrapper_tool_calls.cpp +++ /dev/null @@ -1,236 +0,0 @@ -#include "wrapper_tool_calls.h" -#include "wrapper_token_text.h" - -#include "llama.cpp/common/chat-auto-parser.h" -#include "llama.cpp/common/chat-auto-parser-helpers.h" -#include "llama.cpp/common/chat.h" -#include "llama.cpp/common/json.h" -#include "llama.cpp/include/llama.h" -#include "wrapper_report_current_exception.h" -#include "wrapper_utils.h" - -#include -#include - -using wrapper_helpers::token_text_or_empty; - -namespace { - -struct tool_call_probe_params { - template_params without_tool_calls; - template_params with_tool_calls; -}; - -auto build_tool_call_probe_params() -> tool_call_probe_params { - common_json const user_msg = { - { "role", "user" }, - { "content", "Please use the tool" } - }; - common_json const assistant_no_tools = { - { "role", "assistant" }, - { "content", "Sure, calling." } - }; - common_json const first_tool_call = { - { "id", "call_001" }, - { "type", "function" }, - { "function", { - { "name", "tool_first" }, - { "arguments", { - { "arg_first", "XXXX" }, - { "arg_second", "YYYY" }, - }} - }} - }; - common_json const assistant_with_tools = { - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", common_json::array({ first_tool_call }) } - }; - common_json const tool_definition = { - { "type", "function" }, - { "function", { - { "name", "tool_first" }, - { "description", "First test tool" }, - { "parameters", { - { "type", "object" }, - { "properties", { - { "arg_first", { { "type", "string" }, { "description", "first arg" } } }, - { "arg_second", { { "type", "string" }, { "description", "second arg" } } }, - }}, - { "required", common_json::array({ "arg_first", "arg_second" }) }, - }} - }} - }; - - tool_call_probe_params probe; - probe.without_tool_calls.messages = common_json::array({ user_msg, assistant_no_tools }); - probe.without_tool_calls.tools = common_json::array({ tool_definition }); - probe.without_tool_calls.add_generation_prompt = false; - probe.without_tool_calls.enable_thinking = true; - - probe.with_tool_calls = probe.without_tool_calls; - probe.with_tool_calls.messages = common_json::array({ user_msg, assistant_with_tools }); - - return probe; -} - -auto detect_tool_call_haystack( - const common_chat_template & tmpl, - const autoparser::analyze_reasoning & reasoning) -> std::string { - tool_call_probe_params const probe = build_tool_call_probe_params(); - - std::string const output_no_tools = autoparser::apply_template(tmpl, probe.without_tool_calls); - std::string const output_with_tools = autoparser::apply_template(tmpl, probe.with_tool_calls); - - if (output_no_tools.empty() || output_with_tools.empty()) { - return {}; - } - - diff_split const diff = calculate_diff_split(output_no_tools, output_with_tools); - std::string haystack = diff.right; - - auto remove_first = [&haystack](const std::string & needle) -> void { - if (needle.empty()) { - return; - } - auto pos = haystack.find(needle); - if (pos != std::string::npos) { - haystack.erase(pos, needle.length()); - } - }; - - remove_first(reasoning.start); - remove_first(reasoning.end); - - return haystack; -} - -} // namespace - -extern "C" auto llama_rs_compute_tool_call_haystack( - const struct llama_model * model, - char ** out_haystack, - char ** out_error) -> llama_rs_compute_tool_call_haystack_status { - if (out_haystack != nullptr) { - *out_haystack = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_MODEL_ARG; - } - if (out_haystack == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_HAYSTACK_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_ERROR_ARG; - } - - try { - const char * tmpl_src = llama_model_chat_template(model, nullptr); - if (tmpl_src == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_CHAT_TEMPLATE; - } - - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(tmpl_src, bos_token, eos_token); - auto jinja_caps = tmpl.original_caps(); - autoparser::analyze_reasoning const reasoning(tmpl, jinja_caps.supports_tool_calls); - - std::string const haystack = detect_tool_call_haystack(tmpl, reasoning); - if (haystack.empty()) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK; - } - - char * haystack_dup = llama_rs_dup_string(haystack); - if (haystack_dup == nullptr) { - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED; - } - - *out_haystack = haystack_dup; - - return LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED); - } -} - -extern "C" auto llama_rs_diagnose_tool_call_synthetic_renders( - const struct llama_model * model, - char ** out_no_tools, - char ** out_with_tools, - char ** out_error) -> llama_rs_diagnose_tool_call_synthetic_renders_status { - if (out_no_tools != nullptr) { - *out_no_tools = nullptr; - } - if (out_with_tools != nullptr) { - *out_with_tools = nullptr; - } - if (out_error != nullptr) { - *out_error = nullptr; - } - if (model == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_MODEL_ARG; - } - if (out_no_tools == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_NO_TOOLS_ARG; - } - if (out_with_tools == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_WITH_TOOLS_ARG; - } - if (out_error == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_ERROR_ARG; - } - - try { - const char * tmpl_src = llama_model_chat_template(model, nullptr); - if (tmpl_src == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_CHAT_TEMPLATE; - } - - const llama_vocab * vocab = llama_model_get_vocab(model); - if (vocab == nullptr) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_VOCAB; - } - - std::string const bos_token = token_text_or_empty(vocab, llama_vocab_bos(vocab)); - std::string const eos_token = token_text_or_empty(vocab, llama_vocab_eos(vocab)); - - common_chat_template const tmpl(tmpl_src, bos_token, eos_token); - - tool_call_probe_params const probe = build_tool_call_probe_params(); - - std::string const output_a = autoparser::apply_template(tmpl, probe.without_tool_calls); - std::string const output_b = autoparser::apply_template(tmpl, probe.with_tool_calls); - - std::unique_ptr a_dup(llama_rs_dup_string(output_a)); - std::unique_ptr b_dup(llama_rs_dup_string(output_b)); - - if ((a_dup == nullptr) || (b_dup == nullptr)) { - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED; - } - - *out_no_tools = a_dup.release(); - *out_with_tools = b_dup.release(); - - return LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_OK; - } catch (...) { - return llama_rs_report_current_exception( - out_error, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_THREW_CXX_EXCEPTION, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED); - } -} diff --git a/llama-cpp-bindings-sys/wrapper_tool_calls.h b/llama-cpp-bindings-sys/wrapper_tool_calls.h deleted file mode 100644 index fce2c366c..000000000 --- a/llama-cpp-bindings-sys/wrapper_tool_calls.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include "llama.cpp/include/llama.h" -#include "wrapper_utils.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum llama_rs_compute_tool_call_haystack_status { - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK = 0, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_MODEL_ARG, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_HAYSTACK_ARG, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_ERROR_ARG, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_CHAT_TEMPLATE, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_VOCAB, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_compute_tool_call_haystack_status; - -llama_rs_compute_tool_call_haystack_status llama_rs_compute_tool_call_haystack( - const struct llama_model * model, - char ** out_haystack, - char ** out_error); - -typedef enum llama_rs_diagnose_tool_call_synthetic_renders_status { - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_OK = 0, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_MODEL_ARG, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_NO_TOOLS_ARG, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_WITH_TOOLS_ARG, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_ERROR_ARG, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_CHAT_TEMPLATE, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_VOCAB, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_OUT_OF_MEMORY, - LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_THREW_CXX_EXCEPTION, -} llama_rs_diagnose_tool_call_synthetic_renders_status; - -llama_rs_diagnose_tool_call_synthetic_renders_status llama_rs_diagnose_tool_call_synthetic_renders( - const struct llama_model * model, - char ** out_no_tools, - char ** out_with_tools, - char ** out_error); - -#ifdef __cplusplus -} -#endif diff --git a/llama-cpp-bindings-sys/wrapper_utils.h b/llama-cpp-bindings-sys/wrapper_utils.h index 43ee99ce6..7e45f1a3f 100644 --- a/llama-cpp-bindings-sys/wrapper_utils.h +++ b/llama-cpp-bindings-sys/wrapper_utils.h @@ -27,4 +27,9 @@ static inline char * llama_rs_dup_string(const char * value) noexcept { return llama_rs_dup_bytes(value, std::strlen(value)); } +static inline const char * llama_rs_string_view(const std::string & text, size_t * out_length) noexcept { + *out_length = text.size(); + return text.data(); +} + #endif diff --git a/llama-cpp-bindings-tests/Cargo.toml b/llama-cpp-bindings-tests/Cargo.toml index 1ec14b151..e873ffe65 100644 --- a/llama-cpp-bindings-tests/Cargo.toml +++ b/llama-cpp-bindings-tests/Cargo.toml @@ -10,10 +10,16 @@ autotests = false [dependencies] anyhow = { workspace = true } encoding_rs = { workspace = true } +inventory = { workspace = true } llama-cpp-bindings = { workspace = true } llama-cpp-bindings-sys = { workspace = true } llama-cpp-bindings-types = { workspace = true } +llama-cpp-fault-injection = { workspace = true } +llama-cpp-ffi-status = { workspace = true } +llama-cpp-gbnf = { workspace = true } llama-cpp-test-harness = { workspace = true } +llama-cpp-test-harness-macros = { workspace = true } +llama-cpp-test-model-provisioning = { workspace = true } serde_json = { workspace = true } toktrie = { workspace = true } diff --git a/llama-cpp-bindings-tests/src/build_user_prompt_with_media_marker.rs b/llama-cpp-bindings-tests/src/build_user_prompt_with_media_marker.rs deleted file mode 100644 index 2b3fabf7d..000000000 --- a/llama-cpp-bindings-tests/src/build_user_prompt_with_media_marker.rs +++ /dev/null @@ -1,16 +0,0 @@ -use anyhow::Result; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::mtmd_default_marker; - -/// # Errors -/// -/// Forwards chat-template lookup, message construction, and template application errors. -pub fn build_user_prompt_with_media_marker(model: &LlamaModel, question: &str) -> Result { - let marker = mtmd_default_marker()?; - let user_content = format!("{marker}{question}"); - let chat_template = model.chat_template(None)?; - let messages = [LlamaChatMessage::new("user".to_string(), user_content)?]; - - Ok(model.apply_chat_template(&chat_template, &messages, true, true)?) -} diff --git a/llama-cpp-bindings-tests/src/chat_answer.rs b/llama-cpp-bindings-tests/src/chat_answer.rs new file mode 100644 index 000000000..6cce95f06 --- /dev/null +++ b/llama-cpp-bindings-tests/src/chat_answer.rs @@ -0,0 +1,104 @@ +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings_types::token_usage::TokenUsage; + +use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; + +fn thinking_tags(session: &ChatSession<'_>) -> Vec { + let mut tags = session + .thinking_end_tags() + .expect("the session's thinking end tags must be UTF-8"); + + tags.push( + session + .thinking_start_tag() + .expect("the session's thinking start tag must be UTF-8"), + ); + + tags +} + +/// A generated answer to a chat session, with what the classifier attributed to each section. +#[derive(Debug)] +pub struct ChatAnswer { + pub outcome: ClassifySampleLoopOutcome, + pub usage: TokenUsage, +} + +impl ChatAnswer { + /// Asserts that every generated token is content, and that no thinking tag of `session` + /// reached the content stream. + /// + /// # Panics + /// + /// Panics when the answer reasons, holds a token of no section, or leaks a thinking tag. + pub fn assert_content_without_reasoning(&self, session: &ChatSession<'_>) { + let Self { outcome, usage } = self; + + assert_ne!(outcome.generated.text(), "", "{outcome:?}"); + assert!(outcome.observed_content > 0, "{outcome:?}"); + assert_eq!(outcome.observed_reasoning, 0, "{outcome:?}"); + assert_eq!(outcome.observed_undeterminable, 0, "{outcome:?}"); + assert_eq!(usage.reasoning_tokens, 0, "{usage:?}"); + assert_eq!(usage.undeterminable_tokens, 0, "{usage:?}"); + assert_eq!( + usage.completion_tokens(), + outcome.observed_content, + "{usage:?}" + ); + + for tag in thinking_tags(session) { + assert!( + !outcome.content_stream.contains(&tag), + "the content stream leaked the thinking tag {tag:?}; {outcome:?}" + ); + } + } + + /// Asserts that the answer reasons, and that the reasoning and content streams are exactly + /// what the session's parser extracts, apart from `prompted_reasoning`, the reasoning text + /// the generation prompt itself holds, which the parser shows in front of the first + /// generated text. + /// + /// # Panics + /// + /// Panics when the parser fails on or does not recognize the answer, or when the answer + /// does not reason, holds a token of no section, leaks a thinking tag, or streams text the + /// parser does not extract. + pub fn assert_reasoning_matches_parse( + &self, + session: &ChatSession<'_>, + prompted_reasoning: &str, + ) { + let Self { outcome, usage } = self; + let parsed = session + .parse(&outcome.generated, ChatOutputCompleteness::Finished) + .expect("the session's parser must parse the generated answer") + .into_recognized() + .expect("the session's parser must recognize the generated answer"); + + assert!(outcome.observed_reasoning > 0, "{outcome:?}"); + assert_eq!(outcome.observed_undeterminable, 0, "{outcome:?}"); + assert_eq!( + usage.reasoning_tokens, outcome.observed_reasoning, + "{usage:?}" + ); + assert_eq!(usage.undeterminable_tokens, 0, "{usage:?}"); + assert_eq!( + usage.completion_tokens(), + outcome.observed_content + outcome.observed_reasoning, + "{usage:?}" + ); + assert_ne!(parsed.reasoning_content, "", "{outcome:?}"); + assert_eq!( + format!("{prompted_reasoning}{}", outcome.reasoning_stream), + parsed.reasoning_content + ); + assert_eq!(outcome.content_stream, parsed.content); + + for tag in thinking_tags(session) { + assert!(!outcome.reasoning_stream.contains(&tag), "{outcome:?}"); + assert!(!outcome.content_stream.contains(&tag), "{outcome:?}"); + } + } +} diff --git a/llama-cpp-bindings-tests/src/chat_answer_generation.rs b/llama-cpp-bindings-tests/src/chat_answer_generation.rs new file mode 100644 index 000000000..d5402ca0f --- /dev/null +++ b/llama-cpp-bindings-tests/src/chat_answer_generation.rs @@ -0,0 +1,65 @@ +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; + +use crate::chat_answer::ChatAnswer; +use crate::classify_sample_loop::ClassifySampleLoop; + +/// Generates the answer to a chat session from its tokenized prompt, classifying every token. +pub struct ChatAnswerGeneration<'borrow, 'model> { + pub fixture: &'borrow LlamaFixture<'model>, + pub session: &'borrow ChatSession<'model>, + pub sampler: &'borrow mut LlamaSampler, + pub max_generated_tokens: i32, +} + +impl ChatAnswerGeneration<'_, '_> { + /// # Panics + /// + /// Panics when creating the context, tokenizing or decoding the prompt, or generating the + /// answer fails. + #[must_use] + pub fn run(self) -> ChatAnswer { + let Self { + fixture, + session, + sampler, + max_generated_tokens, + } = self; + let mut context = fixture + .build_context() + .expect("the fixture's context must build"); + let prompt_tokens = session + .tokenize_prompt() + .expect("the session's prompt must tokenize"); + let mut classifier = SampledTokenClassifier::for_chat_session(session); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + classifier + .feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false) + .expect("the prompt must fit the batch"); + context + .decode(&batch) + .expect("decoding the prompt must succeed"); + classifier.commit_prompt_tokens(); + + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler, + context: &mut context, + batch: &mut batch, + initial_position, + max_generated_tokens, + } + .run(); + + ChatAnswer { + outcome, + usage: classifier.into_usage(), + } + } +} diff --git a/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs b/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs index 8b07191e8..0d971141a 100644 --- a/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs +++ b/llama-cpp-bindings-tests/src/chunk_token_breakdown.rs @@ -1,7 +1,5 @@ -use anyhow::Context; -use anyhow::Result; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; pub struct ChunkTokenBreakdown { pub text: u64, @@ -10,27 +8,34 @@ pub struct ChunkTokenBreakdown { } impl ChunkTokenBreakdown { - /// # Errors + /// # Panics /// - /// Forwards chunk access and chunk-type classification errors. - pub fn from_chunks(chunks: &MtmdInputChunks) -> Result { + /// Panics when a chunk below the chunk count is missing or reports a type unknown to the + /// bindings. + #[must_use] + pub fn from_chunks(chunks: &MtmdInputChunks) -> Self { let mut breakdown = Self { text: 0, image: 0, audio: 0, }; + for index in 0..chunks.len() { let chunk = chunks .get(index) - .with_context(|| format!("chunk index {index} is missing"))?; - let n_tokens = u64::try_from(chunk.n_tokens())?; - match chunk.chunk_type()? { + .expect("every index below the chunk count must hold a chunk"); + let n_tokens = u64::try_from(chunk.n_tokens()).expect("a chunk's token count fits u64"); + + match chunk + .chunk_type() + .expect("every chunk must have a known type") + { MtmdInputChunkType::Text => breakdown.text += n_tokens, MtmdInputChunkType::Image => breakdown.image += n_tokens, MtmdInputChunkType::Audio => breakdown.audio += n_tokens, } } - Ok(breakdown) + breakdown } } diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop.rs b/llama-cpp-bindings-tests/src/classify_sample_loop.rs index 941eed4db..16805f075 100644 --- a/llama-cpp-bindings-tests/src/classify_sample_loop.rs +++ b/llama-cpp-bindings-tests/src/classify_sample_loop.rs @@ -1,82 +1,45 @@ -use anyhow::Result; -use llama_cpp_bindings::GenerationProgress; -use llama_cpp_bindings::SampledTokenSection; use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::generation_progress::GenerationProgress; use llama_cpp_bindings::llama_batch::LlamaBatch; use llama_cpp_bindings::sampled_token::SampledToken; use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; -pub struct ClassifySampleLoop<'borrow, 'model, 'tokens> { - pub classifier: &'borrow mut SampledTokenClassifier<'model>, +use crate::classify_sample_loop_outcome::ClassifySampleLoopOutcome; + +pub struct ClassifySampleLoop<'borrow, 'session, 'model> { + pub classifier: &'borrow mut SampledTokenClassifier<'session, 'model>, pub sampler: &'borrow mut LlamaSampler, pub context: &'borrow mut LlamaContext<'model>, - pub batch: &'borrow mut LlamaBatch<'tokens>, + pub batch: &'borrow mut LlamaBatch, pub initial_position: i32, pub max_generated_tokens: i32, } -#[derive(Debug, Default)] -pub struct ClassifySampleLoopOutcome { - pub generated_raw: String, - pub content_stream: String, - pub reasoning_stream: String, - pub observed_content: u64, - pub observed_reasoning: u64, - pub observed_tool_call: u64, - pub observed_undeterminable: u64, - pub eog_seen: bool, -} - -impl ClassifySampleLoopOutcome { - const fn record_end_of_generation(&mut self, section: SampledTokenSection) { - self.eog_seen = true; - - match section { - SampledTokenSection::Content => self.observed_content += 1, - SampledTokenSection::Reasoning => self.observed_reasoning += 1, - SampledTokenSection::ToolCall => self.observed_tool_call += 1, - SampledTokenSection::Pending => self.observed_undeterminable += 1, - } - } - - fn record_outcome(&mut self, ingest: &IngestOutcome) { - self.generated_raw.push_str(ingest.piece.raw()); - - match ingest.sampled_token { - SampledToken::Content(_) => { - self.observed_content += 1; - self.content_stream.push_str(ingest.piece.visible()); - } - SampledToken::Reasoning(_) => { - self.observed_reasoning += 1; - self.reasoning_stream.push_str(ingest.piece.visible()); - } - SampledToken::ToolCall(_) => self.observed_tool_call += 1, - SampledToken::Undeterminable(_) => self.observed_undeterminable += 1, - } - } -} - impl ClassifySampleLoop<'_, '_, '_> { - /// # Errors - /// Forwards [`SampledTokenClassifier::sample`] / [`LlamaContext::decode`] / - /// [`LlamaBatch::add`] errors verbatim. Stops on the end of generation, on - /// `max_generated_tokens` exhaustion, or on the first error. - pub fn run(self) -> Result { + /// Samples and classifies tokens until the end of generation or until + /// `max_generated_tokens` are generated. + /// + /// # Panics + /// + /// Panics when sampling, classifying, batching, or decoding a token fails. + #[must_use] + pub fn run(self) -> ClassifySampleLoopOutcome { let mut outcome = ClassifySampleLoopOutcome::default(); let mut ingest_outcomes = Vec::new(); let mut position = self.initial_position; let max_position = position + self.max_generated_tokens; while position < max_position { - let sampled = self.classifier.sample( - self.sampler, - self.context, - self.batch.n_tokens() - 1, - &mut ingest_outcomes, - )?; + let sampled = self + .classifier + .sample( + self.sampler, + self.context, + self.batch.n_tokens() - 1, + &mut ingest_outcomes, + ) + .expect("sampling and classifying the next token must succeed"); if sampled.progress == GenerationProgress::Ended { outcome.record_end_of_generation(self.classifier.current_section()); @@ -86,80 +49,23 @@ impl ClassifySampleLoop<'_, '_, '_> { self.batch.clear(); self.batch - .add(&SampledToken::Content(sampled.token), position, &[0], true)?; + .add(&SampledToken::Content(sampled.token), position, &[0], true) + .expect("the sampled token must fit the batch"); position += 1; - self.context.decode(self.batch)?; + self.context + .decode(self.batch) + .expect("decoding the sampled token must succeed"); } - self.classifier.finish(&mut ingest_outcomes); + self.classifier + .finish(&mut ingest_outcomes) + .expect("finishing the classification must succeed"); for ingest_outcome in &ingest_outcomes { outcome.record_outcome(ingest_outcome); } - Ok(outcome) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings::SampledTokenSection; - use llama_cpp_bindings::TokenPiece; - use llama_cpp_bindings::ingest_outcome::IngestOutcome; - use llama_cpp_bindings::sampled_token::SampledToken; - use llama_cpp_bindings::token::LlamaToken; - - use super::ClassifySampleLoopOutcome; - - #[test] - fn records_a_tool_call_token_without_streaming_it() { - let mut outcome = ClassifySampleLoopOutcome::default(); - - outcome.record_outcome(&IngestOutcome { - sampled_token: SampledToken::ToolCall(LlamaToken(42)), - piece: TokenPiece::Visible("{".to_owned()), - }); - - assert_eq!(outcome.observed_tool_call, 1); - assert_eq!(outcome.generated_raw, "{"); - assert_eq!(outcome.content_stream, ""); - } - - #[test] - fn streams_the_visible_part_of_a_reasoning_token() { - let mut outcome = ClassifySampleLoopOutcome::default(); - - outcome.record_outcome(&IngestOutcome { - sampled_token: SampledToken::Reasoning(LlamaToken(7)), - piece: TokenPiece::Visible("thinking".to_owned()), - }); - - assert_eq!(outcome.observed_reasoning, 1); - assert_eq!(outcome.reasoning_stream, "thinking"); - } - - #[test] - fn counts_an_undeterminable_token_without_streaming_it() { - let mut outcome = ClassifySampleLoopOutcome::default(); - - outcome.record_outcome(&IngestOutcome { - sampled_token: SampledToken::Undeterminable(LlamaToken(9)), - piece: TokenPiece::Visible("ignored".to_owned()), - }); - - assert_eq!(outcome.observed_undeterminable, 1); - assert_eq!(outcome.content_stream, ""); - assert_eq!(outcome.reasoning_stream, ""); - } - - #[test] - fn counts_the_end_of_generation_in_the_section_it_ended() { - let mut outcome = ClassifySampleLoopOutcome::default(); - - outcome.record_end_of_generation(SampledTokenSection::Content); - - assert!(outcome.eog_seen); - assert_eq!(outcome.observed_content, 1); + outcome } } diff --git a/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs b/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs new file mode 100644 index 000000000..50ac78509 --- /dev/null +++ b/llama-cpp-bindings-tests/src/classify_sample_loop_outcome.rs @@ -0,0 +1,132 @@ +use llama_cpp_bindings::chat_output::ChatOutput; +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_section::SampledTokenSection; + +#[derive(Debug, Default)] +pub struct ClassifySampleLoopOutcome { + pub generated: ChatOutput, + pub content_stream: String, + pub reasoning_stream: String, + pub observed_content: u64, + pub observed_reasoning: u64, + pub observed_tool_call: u64, + pub observed_undeterminable: u64, + pub eog_seen: bool, +} + +impl ClassifySampleLoopOutcome { + pub const fn record_end_of_generation(&mut self, section: SampledTokenSection) { + self.eog_seen = true; + + match section { + SampledTokenSection::Content => self.observed_content += 1, + SampledTokenSection::Reasoning => self.observed_reasoning += 1, + SampledTokenSection::ToolCall => self.observed_tool_call += 1, + SampledTokenSection::Pending => self.observed_undeterminable += 1, + } + } + + pub fn record_outcome(&mut self, ingest: &IngestOutcome) { + self.generated + .push(ingest.sampled_token.token(), ingest.piece.raw()); + + match ingest.sampled_token { + SampledToken::Content(_) => { + self.observed_content += 1; + self.content_stream.push_str(ingest.piece.visible()); + } + SampledToken::Reasoning(_) => { + self.observed_reasoning += 1; + self.reasoning_stream.push_str(ingest.piece.visible()); + } + SampledToken::ToolCall(_) => self.observed_tool_call += 1, + SampledToken::Undeterminable(_) => self.observed_undeterminable += 1, + } + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_bindings::ingest_outcome::IngestOutcome; + use llama_cpp_bindings::sampled_token::SampledToken; + use llama_cpp_bindings::sampled_token_section::SampledTokenSection; + use llama_cpp_bindings::token::LlamaToken; + use llama_cpp_bindings::token_piece::TokenPiece; + + use super::ClassifySampleLoopOutcome; + + #[test] + fn records_a_tool_call_token_without_streaming_it() { + let mut outcome = ClassifySampleLoopOutcome::default(); + + outcome.record_outcome(&IngestOutcome { + sampled_token: SampledToken::ToolCall(LlamaToken(42)), + piece: TokenPiece::Visible("{".to_owned()), + }); + + assert_eq!(outcome.observed_tool_call, 1); + assert_eq!(outcome.generated.text(), "{"); + assert_eq!(outcome.content_stream, ""); + } + + #[test] + fn streams_the_visible_part_of_a_reasoning_token() { + let mut outcome = ClassifySampleLoopOutcome::default(); + + outcome.record_outcome(&IngestOutcome { + sampled_token: SampledToken::Reasoning(LlamaToken(7)), + piece: TokenPiece::Visible("thinking".to_owned()), + }); + + assert_eq!(outcome.observed_reasoning, 1); + assert_eq!(outcome.reasoning_stream, "thinking"); + } + + #[test] + fn counts_an_undeterminable_token_without_streaming_it() { + let mut outcome = ClassifySampleLoopOutcome::default(); + + outcome.record_outcome(&IngestOutcome { + sampled_token: SampledToken::Undeterminable(LlamaToken(9)), + piece: TokenPiece::Visible("ignored".to_owned()), + }); + + assert_eq!(outcome.observed_undeterminable, 1); + assert_eq!(outcome.content_stream, ""); + assert_eq!(outcome.reasoning_stream, ""); + } + + #[test] + fn counts_the_end_of_generation_in_the_section_it_ended() { + let counts = [ + SampledTokenSection::Content, + SampledTokenSection::Reasoning, + SampledTokenSection::ToolCall, + SampledTokenSection::Pending, + ] + .map(|section| { + let mut outcome = ClassifySampleLoopOutcome::default(); + + outcome.record_end_of_generation(section); + + [ + u64::from(outcome.eog_seen), + outcome.observed_content, + outcome.observed_reasoning, + outcome.observed_tool_call, + outcome.observed_undeterminable, + ] + }); + + assert_eq!( + counts, + [ + [1, 1, 0, 0, 0], + [1, 0, 1, 0, 0], + [1, 0, 0, 1, 0], + [1, 0, 0, 0, 1], + ] + ); + } +} diff --git a/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs new file mode 100644 index 000000000..941a59811 --- /dev/null +++ b/llama-cpp-bindings-tests/src/closed_reasoning_chat_session.rs @@ -0,0 +1,47 @@ +use llama_cpp_bindings::chat_assistant_prefill::ChatAssistantPrefill; +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; + +const PREFILLED_REASONING: &str = "\nThe question is simple, so I answer it directly.\n"; +const PREFILLED_CONTENT: &str = "\n\n"; + +/// A chat session for one user turn whose answer continues prefilled content. +/// +/// The prefill closes a reasoning block before the content begins, so the model writes only +/// content even when its template cannot disable thinking. +/// +/// # Panics +/// +/// Panics when the question holds a NUL byte or the model's template cannot serve the request. +#[must_use] +pub fn closed_reasoning_chat_session<'model>( + model: &'model LlamaModel, + question: &str, + tools_json: &str, +) -> ChatSession<'model> { + ChatSession::new( + model, + model + .chat_templates() + .expect("the model must have chat templates"), + &[ + LlamaChatMessage::new("user".to_owned(), question.to_owned()) + .expect("the question must hold no NUL byte"), + ], + tools_json, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { + reasoning_content: PREFILLED_REASONING.to_owned(), + content: PREFILLED_CONTENT.to_owned(), + }), + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + ) + .expect("the model's template must serve a prefilled answer") +} diff --git a/llama-cpp-bindings-tests/src/context_after_decoding_text.rs b/llama-cpp-bindings-tests/src/context_after_decoding_text.rs new file mode 100644 index 000000000..3e8d5d086 --- /dev/null +++ b/llama-cpp-bindings-tests/src/context_after_decoding_text.rs @@ -0,0 +1,41 @@ +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; + +const TEXT: &str = "The parcel arrived on the last working day of the week."; + +/// A context with `pooling_type` that has decoded one sentence as sequence 0. +/// +/// # Panics +/// +/// Panics when the fixture's context cannot be created or the sentence cannot be decoded in it. +#[must_use] +pub fn context_after_decoding_text<'fixture>( + fixture: &'fixture LlamaFixture<'_>, + pooling_type: LlamaPoolingType, +) -> LlamaContext<'fixture> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_pooling_type(pooling_type), + ) + .expect("the fixture's context must be creatable"); + let tokens = fixture + .model + .str_to_token(TEXT, AddBos::Always, ParseSpecialTokens::Never) + .expect("the sentence must tokenize"); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch + .add_sequence(&tokens, 0, false) + .expect("the sentence must fit one batch"); + context.decode(&batch).expect("the sentence must decode"); + + context +} diff --git a/llama-cpp-bindings-tests/src/evaluated_multimodal_usage.rs b/llama-cpp-bindings-tests/src/evaluated_multimodal_usage.rs new file mode 100644 index 000000000..aa17a3440 --- /dev/null +++ b/llama-cpp-bindings-tests/src/evaluated_multimodal_usage.rs @@ -0,0 +1,8 @@ +use llama_cpp_bindings_types::token_usage::TokenUsage; + +use crate::chunk_token_breakdown::ChunkTokenBreakdown; + +pub struct EvaluatedMultimodalUsage { + pub usage: TokenUsage, + pub chunk_breakdown: ChunkTokenBreakdown, +} diff --git a/llama-cpp-bindings-tests/src/fixtures_dir.rs b/llama-cpp-bindings-tests/src/fixtures_dir.rs index 4683561fd..d92932e91 100644 --- a/llama-cpp-bindings-tests/src/fixtures_dir.rs +++ b/llama-cpp-bindings-tests/src/fixtures_dir.rs @@ -7,18 +7,14 @@ pub fn fixtures_dir() -> PathBuf { #[cfg(test)] mod tests { - use std::path::PathBuf; - use super::fixtures_dir; #[test] - fn resolves_to_the_fixtures_directory_inside_the_manifest() { - let expected = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures"); - - assert_eq!(fixtures_dir(), expected); - assert!( - fixtures_dir().is_dir(), - "the fixtures directory the multimodal tests read from must exist" + fn resolves_to_the_directory_holding_every_media_fixture() { + assert_eq!( + ["llamas.jpg", "orange_cat.wav", "quick_brown_fox.wav"] + .map(|file_name| fixtures_dir().join(file_name).is_file()), + [true, true, true] ); } } diff --git a/llama-cpp-bindings-tests/src/get_weather_tools_json.rs b/llama-cpp-bindings-tests/src/get_weather_tools_json.rs new file mode 100644 index 000000000..71ce47531 --- /dev/null +++ b/llama-cpp-bindings-tests/src/get_weather_tools_json.rs @@ -0,0 +1,16 @@ +pub const GET_WEATHER_TOOLS_JSON: &str = r#"[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name"} + }, + "required": ["location"] + } + } + } +]"#; diff --git a/llama-cpp-bindings-tests/src/ingested_generation.rs b/llama-cpp-bindings-tests/src/ingested_generation.rs new file mode 100644 index 000000000..cb3f3193e --- /dev/null +++ b/llama-cpp-bindings-tests/src/ingested_generation.rs @@ -0,0 +1,43 @@ +use llama_cpp_bindings::generation_progress::GenerationProgress; +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; + +/// Feeds the tokens of `generated_text` to `classifier` as if the model generated them, then +/// ends the generation with the model's end-of-sequence token. +/// +/// # Panics +/// +/// Panics when tokenizing or classifying fails, or when the classifier ends the generation +/// before the end-of-sequence token, or does not end it on that token. +pub fn ingested_generation( + classifier: &mut SampledTokenClassifier<'_, '_>, + model: &LlamaModel, + generated_text: &str, +) -> Vec { + let mut outcomes = Vec::new(); + + for token in model + .str_to_token(generated_text, AddBos::Never, ParseSpecialTokens::Always) + .expect("the generated text must tokenize") + { + assert_eq!( + classifier.ingest(token, &mut outcomes), + Ok(GenerationProgress::Continues) + ); + } + + assert_eq!( + classifier.ingest( + model + .token_eos() + .expect("the model must declare an EOS token"), + &mut outcomes + ), + Ok(GenerationProgress::Ended) + ); + + outcomes +} diff --git a/llama-cpp-bindings-tests/src/lib.rs b/llama-cpp-bindings-tests/src/lib.rs index 922330dd0..f2a086267 100644 --- a/llama-cpp-bindings-tests/src/lib.rs +++ b/llama-cpp-bindings-tests/src/lib.rs @@ -1,10 +1,25 @@ -#![cfg_attr( - not(test), - deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) -)] -pub mod build_user_prompt_with_media_marker; +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::panic))] +pub mod chat_answer; +pub mod chat_answer_generation; pub mod chunk_token_breakdown; pub mod classify_sample_loop; +pub mod classify_sample_loop_outcome; +pub mod closed_reasoning_chat_session; +pub mod context_after_decoding_text; +pub mod evaluated_multimodal_usage; pub mod fixtures_dir; -pub mod prime_kv_cache; -pub mod prime_kv_cache_with; +pub mod get_weather_tools_json; +pub mod ingested_generation; +pub mod logits_left_by; +pub mod media_question_chat_session; +pub mod normalize; +pub mod paris_weather_tool_call; +pub mod prime_memory; +pub mod prime_memory_with; +pub mod prompt_question; +pub mod recognized_answer; +pub mod single_token; +pub mod tokenize_prompt_with_synthetic_image; +pub mod two_sequence_context; +pub mod user_turn_chat_session; +pub mod weather_chat_session; diff --git a/llama-cpp-bindings-tests/src/logits_left_by.rs b/llama-cpp-bindings-tests/src/logits_left_by.rs new file mode 100644 index 000000000..519f4a5bc --- /dev/null +++ b/llama-cpp-bindings-tests/src/logits_left_by.rs @@ -0,0 +1,27 @@ +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings::token::data::LlamaTokenData; +use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; + +/// # Panics +/// Panics when `sampler` cannot be applied to the candidates. +#[must_use] +pub fn logits_left_by(sampler: &LlamaSampler, candidates: &[LlamaToken]) -> Vec { + let mut token_data_array = LlamaTokenDataArray::new( + candidates + .iter() + .map(|candidate| LlamaTokenData::new(*candidate, 0.0, 0.0)) + .collect(), + false, + ); + + sampler + .apply(&mut token_data_array) + .expect("the sampler must apply to the candidates"); + + token_data_array + .data + .iter() + .map(LlamaTokenData::logit) + .collect() +} diff --git a/llama-cpp-bindings-tests/src/media_question_chat_session.rs b/llama-cpp-bindings-tests/src/media_question_chat_session.rs new file mode 100644 index 000000000..1798f000b --- /dev/null +++ b/llama-cpp-bindings-tests/src/media_question_chat_session.rs @@ -0,0 +1,32 @@ +use anyhow::Result; + +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; + +use crate::user_turn_chat_session::user_turn_chat_session; + +/// A chat session for one user turn that shows a medium and asks `question` about it, with +/// thinking enabled and no tools. +/// +/// # Errors +/// +/// Forwards message construction and chat session errors. +/// +/// # Panics +/// +/// Panics when llama.cpp's media marker is not UTF-8. +pub fn media_question_chat_session<'model>( + model: &'model LlamaModel, + question: &str, +) -> Result> { + user_turn_chat_session( + model, + &format!( + "{}{question}", + mtmd_default_marker().expect("llama.cpp's media marker must be UTF-8") + ), + "[]", + true, + ) +} diff --git a/llama-cpp-bindings-tests/src/normalize.rs b/llama-cpp-bindings-tests/src/normalize.rs new file mode 100644 index 000000000..a4ac0bbf7 --- /dev/null +++ b/llama-cpp-bindings-tests/src/normalize.rs @@ -0,0 +1,19 @@ +#[must_use] +pub fn normalize(input: &[f32]) -> Vec { + let magnitude = input + .iter() + .fold(0.0, |accumulator, &value| value.mul_add(value, accumulator)) + .sqrt(); + + input.iter().map(|&value| value / magnitude).collect() +} + +#[cfg(test)] +mod tests { + use super::normalize; + + #[test] + fn scales_a_vector_to_unit_length() { + assert_eq!(normalize(&[3.0, 4.0]), vec![0.6, 0.8]); + } +} diff --git a/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs new file mode 100644 index 000000000..a4b7477d9 --- /dev/null +++ b/llama-cpp-bindings-tests/src/paris_weather_tool_call.rs @@ -0,0 +1,15 @@ +use serde_json::json; + +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; + +/// The `get_weather` call for Paris that the tool-call fixtures of every format encode. +#[must_use] +pub fn paris_weather_tool_call() -> ParsedToolCall { + ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "get_weather".to_owned(), + ToolCallArguments::ValidJson(json!({"location": "Paris"})), + ) +} diff --git a/llama-cpp-bindings-tests/src/prime_kv_cache.rs b/llama-cpp-bindings-tests/src/prime_kv_cache.rs deleted file mode 100644 index 1123abd30..000000000 --- a/llama-cpp-bindings-tests/src/prime_kv_cache.rs +++ /dev/null @@ -1,19 +0,0 @@ -use std::num::NonZeroU16; - -use anyhow::Result; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; - -use crate::prime_kv_cache_with::prime_kv_cache_with; - -/// # Errors -/// Forwards tokenization and [`LlamaContext::decode`] errors verbatim. -pub fn prime_kv_cache(fixture: &LlamaFixture<'_>, context: &mut LlamaContext<'_>) -> Result<()> { - prime_kv_cache_with( - fixture, - context, - "Hello world", - BatchCapacity::from(NonZeroU16::MAX), - ) -} diff --git a/llama-cpp-bindings-tests/src/prime_memory.rs b/llama-cpp-bindings-tests/src/prime_memory.rs new file mode 100644 index 000000000..f731145b0 --- /dev/null +++ b/llama-cpp-bindings-tests/src/prime_memory.rs @@ -0,0 +1,12 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; + +use crate::prime_memory_with::prime_memory_with; + +/// # Errors +/// Forwards tokenization and [`LlamaContext::decode`] errors verbatim. +pub fn prime_memory(fixture: &LlamaFixture<'_>, context: &mut LlamaContext<'_>) -> Result<()> { + prime_memory_with(fixture, context, "Hello world") +} diff --git a/llama-cpp-bindings-tests/src/prime_kv_cache_with.rs b/llama-cpp-bindings-tests/src/prime_memory_with.rs similarity index 57% rename from llama-cpp-bindings-tests/src/prime_kv_cache_with.rs rename to llama-cpp-bindings-tests/src/prime_memory_with.rs index 8f8a9ba78..09b47afe5 100644 --- a/llama-cpp-bindings-tests/src/prime_kv_cache_with.rs +++ b/llama-cpp-bindings-tests/src/prime_memory_with.rs @@ -1,24 +1,24 @@ use anyhow::Result; + use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; /// # Errors /// Forwards tokenization, batch assembly, and [`LlamaContext::decode`] errors verbatim. -pub fn prime_kv_cache_with( +pub fn prime_memory_with( fixture: &LlamaFixture<'_>, context: &mut LlamaContext<'_>, text: &str, - batch_capacity: BatchCapacity, ) -> Result<()> { let tokens = fixture .model .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(batch_capacity, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(context)); batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; + context.decode(&batch)?; Ok(()) } diff --git a/llama-cpp-bindings-tests/src/prompt_question.rs b/llama-cpp-bindings-tests/src/prompt_question.rs new file mode 100644 index 000000000..eb121df60 --- /dev/null +++ b/llama-cpp-bindings-tests/src/prompt_question.rs @@ -0,0 +1 @@ +pub const PROMPT_QUESTION: &str = "What animals do you see in this image?"; diff --git a/llama-cpp-bindings-tests/src/recognized_answer.rs b/llama-cpp-bindings-tests/src/recognized_answer.rs new file mode 100644 index 000000000..36fb57893 --- /dev/null +++ b/llama-cpp-bindings-tests/src/recognized_answer.rs @@ -0,0 +1,22 @@ +use llama_cpp_bindings::chat_output::ChatOutput; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; + +/// Parses `answer` with the session's parser. +/// +/// # Panics +/// +/// Panics when parsing fails or the parser does not recognize the answer. +#[must_use] +pub fn recognized_answer( + session: &ChatSession<'_>, + answer: &str, + completeness: ChatOutputCompleteness, +) -> ParsedChatMessage { + session + .parse(&ChatOutput::from(answer), completeness) + .expect("the session's parser must parse the answer") + .into_recognized() + .expect("the session's parser must recognize the answer") +} diff --git a/llama-cpp-bindings-tests/src/single_token.rs b/llama-cpp-bindings-tests/src/single_token.rs new file mode 100644 index 000000000..f7588aab9 --- /dev/null +++ b/llama-cpp-bindings-tests/src/single_token.rs @@ -0,0 +1,17 @@ +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::token::LlamaToken; + +/// # Panics +/// Panics when `text` cannot be tokenized or does not tokenize into exactly one token. +#[must_use] +pub fn single_token(model: &LlamaModel, text: &str) -> LlamaToken { + let [token]: [LlamaToken; 1] = model + .str_to_token(text, AddBos::Never, ParseSpecialTokens::Always) + .expect("the text must tokenize") + .try_into() + .expect("the text must tokenize into exactly one token"); + + token +} diff --git a/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs b/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs new file mode 100644 index 000000000..0c47de71f --- /dev/null +++ b/llama-cpp-bindings-tests/src/tokenize_prompt_with_synthetic_image.rs @@ -0,0 +1,32 @@ +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; + +/// Tokenizes `prompt` with a uniformly grey 64x64 image as its media. +/// +/// # Panics +/// +/// Panics when the fixture declares no multimodal projector, or the image or prompt cannot be +/// tokenized. +#[must_use] +pub fn tokenize_prompt_with_synthetic_image( + fixture: &LlamaFixture<'_>, + prompt: &str, +) -> MtmdInputChunks { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data) + .expect("a 64x64 RGB image must make a bitmap"); + let input_text = MtmdInputText { + text: prompt.to_owned(), + add_special: true, + parse_special: true, + }; + + mtmd_ctx + .tokenize(input_text, &[&bitmap]) + .expect("the prompt and image must tokenize") +} diff --git a/llama-cpp-bindings-tests/src/two_sequence_context.rs b/llama-cpp-bindings-tests/src/two_sequence_context.rs new file mode 100644 index 000000000..9d3955e04 --- /dev/null +++ b/llama-cpp-bindings-tests/src/two_sequence_context.rs @@ -0,0 +1,18 @@ +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::llama_context_load_error::LlamaContextLoadError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; + +/// # Errors +/// Forwards [`LlamaContext::from_model`] errors verbatim. +pub fn two_sequence_context<'model>( + fixture: &LlamaFixture<'model>, +) -> Result, LlamaContextLoadError> { + LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_kv_unified(true) + .with_n_seq_max(2), + ) +} diff --git a/llama-cpp-bindings-tests/src/user_turn_chat_session.rs b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs new file mode 100644 index 000000000..47fd7efb7 --- /dev/null +++ b/llama-cpp-bindings-tests/src/user_turn_chat_session.rs @@ -0,0 +1,42 @@ +use anyhow::Result; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; + +/// A chat session that answers one user turn, offering the tools of `tools_json`. +/// +/// # Errors +/// +/// Forwards message construction and chat session errors. +/// +/// # Panics +/// +/// Panics when the model has no chat templates. +pub fn user_turn_chat_session<'model>( + model: &'model LlamaModel, + content: &str, + tools_json: &str, + enable_thinking: bool, +) -> Result> { + Ok(ChatSession::new( + model, + model + .chat_templates() + .expect("the model must have chat templates"), + &[LlamaChatMessage::new( + "user".to_owned(), + content.to_owned(), + )?], + tools_json, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?) +} diff --git a/llama-cpp-bindings-tests/src/weather_chat_session.rs b/llama-cpp-bindings-tests/src/weather_chat_session.rs new file mode 100644 index 000000000..565fba001 --- /dev/null +++ b/llama-cpp-bindings-tests/src/weather_chat_session.rs @@ -0,0 +1,37 @@ +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; + +use crate::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; + +/// A chat session that asks for the weather in Paris, offering the weather tool with +/// `tool_choice`. +/// +/// # Panics +/// +/// Panics when the model's template cannot serve the request. +#[must_use] +pub fn weather_chat_session(model: &LlamaModel, tool_choice: ChatToolChoice) -> ChatSession<'_> { + ChatSession::new( + model, + model + .chat_templates() + .expect("the model must have chat templates"), + &[LlamaChatMessage::new( + "user".to_owned(), + "What is the weather in Paris?".to_owned(), + ) + .expect("the question holds no NUL byte")], + GET_WEATHER_TOOLS_JSON, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: false, + tool_choice, + }, + ) + .expect("the model's template must serve the weather request") +} diff --git a/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs new file mode 100644 index 000000000..a52c15951 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/approximate_tok_env_eos_selection.rs @@ -0,0 +1,80 @@ +use std::env; +use std::fs; +use std::process; + +use anyhow::Context; +use anyhow::Result; +use toktrie::INVALID_TOKEN; +use toktrie::TokenizerEnv as _; + +use llama_cpp_bindings::gguf_context::GgufContext; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::params::LlamaModelParams; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn approximate_tok_env_falls_back_to_eos_when_eot_unavailable( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let LlamaToken(eos) = fixture + .model + .token_eos() + .context("model declares an EOS token")?; + + assert_eq!( + fixture.model.approximate_tok_env()?.tok_trie().eos_token(), + eos.cast_unsigned() + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn approximate_tok_env_of_a_model_without_eot_and_eos_has_no_eos( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model_without_eos_path = env::temp_dir().join(format!( + "llama_cpp_bindings_{}_model_without_eos.gguf", + process::id() + )); + let mut gguf_context = GgufContext::from_file(fixture.model_path)?; + + gguf_context.remove_key("tokenizer.ggml.eos_token_id")?; + gguf_context.write_to_file(&model_without_eos_path)?; + + let model_without_eos = LlamaModel::load_from_file( + fixture.backend, + &model_without_eos_path, + &LlamaModelParams::default(), + )?; + + fs::remove_file(&model_without_eos_path)?; + + assert_eq!( + model_without_eos + .approximate_tok_env()? + .tok_trie() + .eos_token(), + INVALID_TOKEN + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/backend_device_listing.rs b/llama-cpp-bindings-tests/tests/backend_device_listing.rs new file mode 100644 index 000000000..0e40ff916 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/backend_device_listing.rs @@ -0,0 +1,59 @@ +use llama_cpp_bindings::error::backend_device_text_error::BackendDeviceTextError; +use llama_cpp_bindings::error::llama_backend_device_error::LlamaBackendDeviceError; +use llama_cpp_bindings::error::unknown_backend_device_type::UnknownBackendDeviceType; +use llama_cpp_bindings::llama_backend_device::LlamaBackendDevice; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +fn listing_with_missing_text_after(passing_hits: usize) -> Option { + let _fault = ArmedFault::after_passing( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TEXT_NULL, + passing_hits, + ); + + LlamaBackendDevice::list_all().err() +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_device_text_ggml_does_not_report_refuses_the_listing(_fixture: &LlamaFixture<'_>) { + assert_eq!( + [0, 1, 2].map(listing_with_missing_text_after), + [ + Some(LlamaBackendDeviceError::Name(BackendDeviceTextError::Null)), + Some(LlamaBackendDeviceError::Description( + BackendDeviceTextError::Null + )), + Some(LlamaBackendDeviceError::BackendName( + BackendDeviceTextError::Null + )), + ] + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_device_type_ggml_does_not_define_refuses_the_listing(_fixture: &LlamaFixture<'_>) { + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BACKEND_DEVICE_TYPE_UNKNOWN); + + assert_eq!( + LlamaBackendDevice::list_all().err(), + Some(LlamaBackendDeviceError::UnknownDeviceType( + UnknownBackendDeviceType(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_META + 1) + )) + ); +} diff --git a/llama-cpp-bindings-tests/tests/backend_initialization.rs b/llama-cpp-bindings-tests/tests/backend_initialization.rs deleted file mode 100644 index a71634acb..000000000 --- a/llama-cpp-bindings-tests/tests/backend_initialization.rs +++ /dev/null @@ -1,21 +0,0 @@ -use anyhow::Result; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - void_logs = true, -)] -fn void_logs_suppresses_output(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!( - fixture.model.n_vocab() > 0, - "model must load successfully even when void_logs has been called before init" - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/batch_context_limits.rs b/llama-cpp-bindings-tests/tests/batch_context_limits.rs new file mode 100644 index 000000000..7df696707 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/batch_context_limits.rs @@ -0,0 +1,43 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; +use llama_cpp_bindings::error::decode_error::DecodeError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn decoding_a_batch_built_for_other_context_limits_is_refused( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let context_limits = ContextLimits::of(&context); + let other_limits = ContextLimits { + n_batch: context_limits.n_batch * 2, + ..context_limits + }; + let mut batch = LlamaBatch::new(other_limits); + batch.add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], true)?; + + assert_eq!( + context.decode(&batch), + Err(DecodeError::BatchTransfer( + BatchTransferError::LimitsMismatch { + batch: other_limits, + context: context_limits, + } + )) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs new file mode 100644 index 000000000..dd0473900 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_answer_parsing.rs @@ -0,0 +1,293 @@ +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::chat_message_parse_outcome::ChatMessageParseOutcome; +use llama_cpp_bindings::chat_output::ChatOutput; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; +use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const GENERATED_REASONING: &str = "step one, step two"; +const GENERATED_CONTENT: &str = "actual response"; + +fn parsed_content_after_closed_reasoning(fixture: &LlamaFixture<'_>) -> Result { + let outcome = closed_reasoning_chat_session(fixture.model, "hello", "[]").parse( + &ChatOutput::from("hello world"), + ChatOutputCompleteness::Finished, + )?; + + let ChatMessageParseOutcome::Recognized(parsed) = outcome else { + bail!("expected Recognized for plain content; got Unrecognized"); + }; + + Ok(parsed) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn templates_that_keep_prefill_whitespace_parse_it_into_reasoning_and_content( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + parsed_content_after_closed_reasoning(fixture)?, + ParsedChatMessage::new( + "\n\nhello world".to_owned(), + "\nThe question is simple, so I answer it directly.\n".to_owned(), + Vec::new() + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen_templates_parse_trimmed_prefill_into_reasoning_and_content( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + parsed_content_after_closed_reasoning(fixture)?, + ParsedChatMessage::new( + "hello world".to_owned(), + "The question is simple, so I answer it directly.\n".to_owned(), + Vec::new() + ) + ); + + Ok(()) +} + +fn reasoning_parsed_before_the_end_tag(fixture: &LlamaFixture<'_>) -> Result { + let session = user_turn_chat_session(fixture.model, "hello", "[]", true)?; + + Ok(recognized_answer( + &session, + &format!("{GENERATED_REASONING}{GENERATED_CONTENT}"), + ChatOutputCompleteness::Finished, + )) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_parses_the_reasoning_after_the_newline_its_generation_prompt_opens( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + reasoning_parsed_before_the_end_tag(fixture)?, + ParsedChatMessage::new( + GENERATED_CONTENT.to_owned(), + format!("\n{GENERATED_REASONING}"), + Vec::new() + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parses_the_reasoning_before_the_end_tag_apart_from_the_content( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + reasoning_parsed_before_the_end_tag(fixture)?, + ParsedChatMessage::new( + GENERATED_CONTENT.to_owned(), + GENERATED_REASONING.to_owned(), + Vec::new() + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parses_an_empty_answer_into_an_empty_message(fixture: &LlamaFixture<'_>) -> Result<()> { + let outcome = user_turn_chat_session(fixture.model, "hello", "[]", false)? + .parse(&ChatOutput::from(""), ChatOutputCompleteness::Finished)?; + + let ChatMessageParseOutcome::Recognized(parsed) = outcome else { + bail!("expected Recognized for empty input; got Unrecognized"); + }; + assert_eq!(parsed, ParsedChatMessage::default()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parses_an_answer_with_a_nul_byte_as_content(fixture: &LlamaFixture<'_>) -> Result<()> { + let outcome = user_turn_chat_session(fixture.model, "hello", "[]", false)?.parse( + &ChatOutput::from("hello\0world"), + ChatOutputCompleteness::Finished, + )?; + + let ChatMessageParseOutcome::Recognized(parsed) = outcome else { + bail!("an answer with a NUL byte must be recognized as content"); + }; + + assert_eq!(parsed.content, "hello\0world"); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_chat_session_parses_every_answer_it_is_given(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "hello", "[]", false)?; + + for answer in ["the first answer", "the second answer"] { + let ChatMessageParseOutcome::Recognized(parsed) = + session.parse(&ChatOutput::from(answer), ChatOutputCompleteness::Finished)? + else { + bail!("expected {answer:?} to be recognized as content"); + }; + + assert_eq!(parsed.content, answer); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn every_parsed_field_refuses_text_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let answer = ChatOutput::from( + "\n\n\nParis\n\n\n", + ); + let not_utf8 = String::from_utf8(vec![0xff]) + .expect_err("a lone 0xff byte is not UTF-8") + .utf8_error(); + let fault_points = [ + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_CONTENT_NOT_UTF8, + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_REASONING_CONTENT_NOT_UTF8, + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ID_NOT_UTF8, + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_NAME_NOT_UTF8, + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NOT_UTF8, + ]; + + assert_eq!( + fault_points.map(|fault_point| { + let _fault = ArmedFault::at(fault_point); + + session + .parse(&answer, ChatOutputCompleteness::Finished) + .err() + }), + fault_points.map(|_fault_point| Some(ParseChatMessageError::ParsedTextNotUtf8(not_utf8))) + ); +} diff --git a/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs new file mode 100644 index 000000000..c9b2f8e89 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_prompt_rendering.rs @@ -0,0 +1,168 @@ +use anyhow::Result; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace( + "unsloth/Ministral-3-14B-Reasoning-2512-GGUF", + "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf" + ), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_renders_the_user_message_into_the_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { + let prompt = user_turn_chat_session(fixture.model, "hello", "[]", true)?.prompt()?; + + assert!( + prompt.contains("hello"), + "the model's built-in chat template must render the user message content through the \ + engine; got: {prompt:?}" + ); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_renders_long_messages_in_full(fixture: &LlamaFixture<'_>) -> Result<()> { + let long_content = "a".repeat(2000); + let prompt = user_turn_chat_session(fixture.model, &long_content, "[]", true)?.prompt()?; + + assert!( + prompt.contains(&long_content), + "a long user message must be rendered in full by the engine without truncation" + ); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_generation_prompt_opens_the_assistant_turn_with_thinking( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "hello", "[]", true)?; + + assert_eq!( + session.generation_prompt()?, + "<|im_start|>assistant\n\n" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_omitted_assistant_turn_renders_only_the_messages( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = ChatSession::new( + fixture.model, + fixture.model.chat_templates()?, + &[LlamaChatMessage::new( + "user".to_owned(), + "hello".to_owned(), + )?], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::Omitted, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + + assert_eq!(session.prompt()?, "<|im_start|>user\nhello<|im_end|>\n"); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_protocol.rs b/llama-cpp-bindings-tests/tests/chat_protocol.rs deleted file mode 100644 index 0c4f13722..000000000 --- a/llama-cpp-bindings-tests/tests/chat_protocol.rs +++ /dev/null @@ -1,427 +0,0 @@ -use anyhow::Result; -use anyhow::bail; -use llama_cpp_bindings::ChatMessageParseOutcome; -use llama_cpp_bindings::ChatMessageParser; -use llama_cpp_bindings::ChatTemplateError; -use llama_cpp_bindings::ParseChatMessageError; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings_tests::build_user_prompt_with_media_marker::build_user_prompt_with_media_marker; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) -> Result<()> { - let template = fixture.model.chat_template(None); - assert!(template.is_ok()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace( - "unsloth/Ministral-3-14B-Reasoning-2512-GGUF", - "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf" - ), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn apply_chat_template_produces_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let template = model.chat_template(None)?; - let message = LlamaChatMessage::new("user".to_string(), "hello".to_string())?; - let prompt = model.apply_chat_template(&template, &[message], true, true)?; - - assert!( - prompt.contains("hello"), - "the model's built-in chat template must render the user message content through the \ - engine; got: {prompt:?}" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 64, - n_batch = 64, - n_ubatch = 64, -)] -fn build_user_prompt_surfaces_message_construction_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = build_user_prompt_with_media_marker(fixture.model, "describe\0this"); - - assert!( - result.is_err(), - "an interior null byte in the question must surface a message construction error" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn apply_chat_template_renders_long_messages(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let template = model.chat_template(None)?; - let long_content = "a".repeat(2000); - let message = LlamaChatMessage::new("user".to_string(), long_content.clone())?; - let prompt = model.apply_chat_template(&template, &[message], true, true)?; - - assert!( - prompt.contains(&long_content), - "a long user message must be rendered in full by the engine without truncation" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture - .model - .chat_template(Some("nonexistent_template_name_xyz")); - assert_eq!(result.unwrap_err(), ChatTemplateError::MissingTemplate); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { - let outcome = ChatMessageParser::new(fixture.model, "[]")?.parse("hello world", false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for plain content; got Unrecognized"); - }; - assert_eq!(parsed.tool_calls, Vec::::new()); - assert!(!parsed.is_empty()); - assert!(parsed.content.contains("hello world")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) -> Result<()> { - let input = "step one, step two\n\nactual response"; - let outcome = ChatMessageParser::new(fixture.model, "[]")?.parse(input, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for reasoning section; got Unrecognized"); - }; - assert!( - parsed.reasoning_content.contains("step") || parsed.content.contains("step"), - "neither content nor reasoning contains 'step'; content={:?} reasoning={:?}", - parsed.content, - parsed.reasoning_content - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn parses_empty_input_yields_empty_message(fixture: &LlamaFixture<'_>) -> Result<()> { - let outcome = ChatMessageParser::new(fixture.model, "[]")?.parse("", false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for empty input; got Unrecognized"); - }; - assert_eq!(parsed.tool_calls, Vec::::new()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn parses_with_input_null_byte_reports_the_input_as_the_source( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let result = ChatMessageParser::new(fixture.model, "[]")?.parse("hello\0world", false); - - let Err(ParseChatMessageError::InputContainsNulByte(nul_error)) = result else { - bail!("a NUL byte in the message must be reported against the message"); - }; - - assert_eq!(nul_error.nul_position(), 5); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn chat_message_parser_for_a_tool_missing_its_function_name_fails_to_build( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let result = ChatMessageParser::new( - fixture.model, - r#"[{"type":"function","function":{"description":"reports the weather"}}]"#, - ); - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ToolsParserBuildFailed { - message: "[json.exception.out_of_range.403] key 'name' not found".to_owned(), - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_chat_message_parser_parses_every_message_it_is_given( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let parser = ChatMessageParser::new(fixture.model, "[]")?; - - for message in ["the first answer", "the second answer"] { - let ChatMessageParseOutcome::Recognized(parsed) = parser.parse(message, false)? else { - bail!("expected {message:?} to be recognized as content"); - }; - - assert_eq!(parsed.content, message); - } - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs new file mode 100644 index 000000000..4098cad07 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_session_input_validation.rs @@ -0,0 +1,211 @@ +use std::ffi::CString; + +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::error::chat_session_error::ChatSessionError; +use llama_cpp_bindings::error::new_llama_chat_message_error::NewLlamaChatMessageError; +use llama_cpp_bindings::error::string_to_token_error::StringToTokenError; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 64, + n_batch = 64, + n_ubatch = 64, +)] +fn chat_message_reports_the_nul_byte_position_after_the_media_marker( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let Err(failure) = media_question_chat_session(fixture.model, "describe\0this") else { + bail!("an interior NUL byte in the question must refuse the message"); + }; + let NewLlamaChatMessageError::NulError(nul_error) = + failure.downcast::()?; + + assert_eq!(nul_error.nul_position(), "<__media__>describe".len()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_refuses_a_tool_missing_its_function_name(fixture: &LlamaFixture<'_>) -> Result<()> { + let tools_json = r#"[{"type":"function","function":{"description":"reports the weather"}}]"#; + let result = user_turn_chat_session(fixture.model, "hello", tools_json, false); + let Err(failure) = result else { + bail!("a tool without a function name must refuse the chat session"); + }; + + assert_eq!( + failure.downcast::()?, + ChatSessionError::ToolsInvalid { + message: "Failed to parse tools: [json.exception.out_of_range.403] key 'name' not \ + found; tools = [\n {\n \"type\": \"function\",\n \"function\": {\n \ + \"description\": \"reports the weather\"\n }\n }\n]" + .to_owned(), + } + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_refuses_tools_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + assert_eq!( + user_turn_chat_session(fixture.model, "hello", "[\0]", false) + .err() + .and_then(|failure| failure.downcast::().ok()), + CString::new("[\0]") + .err() + .map(ChatSessionError::RequestContainsNul) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_refuses_a_message_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { + let not_utf8 = CString::new([0xff])?; + let message = LlamaChatMessage { + role: CString::new("user")?, + content: not_utf8.clone(), + }; + + assert_eq!( + ChatSession::new( + fixture.model, + fixture.model.chat_templates()?, + &[message], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + ) + .err(), + not_utf8 + .to_str() + .err() + .map(ChatSessionError::MessageNotUtf8) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn tokenizing_a_prompt_whose_reasoning_prefill_holds_a_nul_character_is_refused( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = ChatSession::new( + fixture.model, + fixture.model.chat_templates()?, + &[LlamaChatMessage::new( + "user".to_owned(), + "hello".to_owned(), + )?], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::ContinueReasoning("a\0b".to_owned()), + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + let Some(nul_error) = session + .prompt_parts()? + .iter() + .find_map(|part| CString::new(part.text()).err()) + else { + bail!("the rendered prompt must hold the reasoning prefill's NUL character"); + }; + + assert_eq!( + session.tokenize_prompt().err(), + Some(ChatSessionError::PromptTokenization( + StringToTokenError::NulError(nul_error) + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn a_conversation_the_template_raises_on_is_refused_with_the_template_message( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let result = ChatSession::new( + fixture.model, + fixture.model.chat_templates()?, + &[ + LlamaChatMessage::new("user".to_owned(), "hello".to_owned())?, + LlamaChatMessage::new("user".to_owned(), "are you there?".to_owned())?, + ], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + ); + + assert_eq!( + result.err(), + Some(ChatSessionError::TemplateCannotServeRequest { + message: "Unable to generate parser for this template. Automatic parser generation \ + failed: \n------------\nWhile executing CallExpression at line 19, column \ + 27 in source:\n...% 2 == 0) -%}↵ {{ raise_exception(\"Conversation \ + roles must alternate user...\n \ + ^\nError: Jinja Exception: Conversation roles must alternate \ + user/assistant/user/assistant/..." + .to_owned() + }) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs new file mode 100644 index 000000000..95125263c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_session_llama_cpp_failures.rs @@ -0,0 +1,246 @@ +use std::str::Utf8Error; + +use llama_cpp_bindings::chat_output::ChatOutput; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::error::chat_session_error::ChatSessionError; +use llama_cpp_bindings::error::chat_templates_error::ChatTemplatesError; +use llama_cpp_bindings::error::grammar_error::GrammarError; +use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; +use llama_cpp_bindings::error::sampler_accept_error::SamplerAcceptError; +use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +fn not_utf8() -> Utf8Error { + String::from_utf8(vec![0xff]) + .expect_err("a lone 0xff byte is not UTF-8") + .utf8_error() +} + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn a_model_without_a_chat_template_has_no_chat_templates(fixture: &LlamaFixture<'_>) { + assert_eq!( + fixture.model.chat_templates().err(), + Some(ChatTemplatesError::ModelHasNoChatTemplate) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn every_prompt_reader_refuses_prompt_text_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_PROMPT_PART_TEXT_NOT_UTF8, + ); + + assert_eq!( + [ + session.prompt_parts().err(), + session.prompt().err(), + session.tokenize_prompt().err(), + ], + [(); 3].map(|()| Some(ChatSessionError::SessionTextNotUtf8(not_utf8()))) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn grammar_triggers_refuse_a_trigger_value_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_VALUE_NOT_UTF8, + ); + + assert_eq!( + [ + session.grammar_triggers().err(), + session.grammar_sampler().err(), + ], + [(); 2].map(|()| Some(ChatSessionError::SessionTextNotUtf8(not_utf8()))) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn grammar_triggers_refuse_a_trigger_type_the_bindings_do_not_recognize( + fixture: &LlamaFixture<'_>, +) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED, + ); + + assert_eq!( + session.grammar_triggers().err(), + Some(ChatSessionError::FfiStatus(FfiStatusError { + operation: "llama_rs_chat_session_grammar_trigger_type", + code: i64::from( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_UNRECOGNIZED + ), + })) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn the_grammar_sampler_refuses_a_grammar_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Required); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_GRAMMAR_NOT_UTF8); + + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::SessionTextNotUtf8(not_utf8())) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn the_grammar_sampler_reports_a_grammar_llama_cpp_cannot_load(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Required); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_THROWS); + + assert!(!session.grammar_is_lazy()); + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::Grammar(GrammarError::Reported { + message: injected_fault_message(), + })) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn the_grammar_sampler_reports_a_generation_prompt_token_the_grammar_cannot_accept( + fixture: &LlamaFixture<'_>, +) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Required); + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_ACCEPT_THROWS); + + assert_ne!(session.generation_prompt_tokens(), []); + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::GrammarPrefill( + SamplerAcceptError::GrammarStateCorrupted { + message: injected_fault_message(), + } + )) + ); +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen2.5-7B-Instruct-GGUF", "qwen2.5-7b-instruct-q2_k.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn the_grammar_sampler_reports_a_trigger_word_llama_cpp_cannot_escape(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS); + + assert!(session.grammar_is_lazy()); + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::Grammar(GrammarError::Reported { + message: injected_fault_message(), + })) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn the_grammar_sampler_reports_a_lazy_grammar_llama_cpp_cannot_load(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_THROWS, + ); + + assert_eq!( + session.grammar_sampler().err(), + Some(ChatSessionError::Grammar(GrammarError::Reported { + message: injected_fault_message(), + })) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn parsing_an_answer_reports_a_parser_llama_cpp_fails(fixture: &LlamaFixture<'_>) { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_PARSE_THROWS); + + assert_eq!( + session + .parse( + &ChatOutput::from("Checking."), + ChatOutputCompleteness::Finished, + ) + .err(), + Some(ParseChatMessageError::Reported { + message: injected_fault_message(), + }) + ); +} diff --git a/llama-cpp-bindings-tests/tests/chat_template_lookup.rs b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs new file mode 100644 index 000000000..100a2a265 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_template_lookup.rs @@ -0,0 +1,101 @@ +use std::ffi::CString; + +use anyhow::Result; + +use llama_cpp_bindings::error::chat_template_error::ChatTemplateError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn default_chat_template_is_not_empty(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_ne!(fixture.model.chat_template(None)?.to_str()?, ""); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) { + let result = fixture + .model + .chat_template(Some("nonexistent_template_name_xyz")); + assert_eq!(result.unwrap_err(), ChatTemplateError::MissingTemplate); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_template_with_a_nul_byte_in_the_name_returns_error(fixture: &LlamaFixture<'_>) { + assert_eq!( + fixture.model.chat_template(Some("tool\0use")).err(), + CString::new("tool\0use") + .err() + .map(ChatTemplateError::NullError) + ); +} diff --git a/llama-cpp-bindings-tests/tests/chat_template_override.rs b/llama-cpp-bindings-tests/tests/chat_template_override.rs new file mode 100644 index 000000000..9cb05a477 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/chat_template_override.rs @@ -0,0 +1,151 @@ +use std::fs; +use std::path::PathBuf; + +use anyhow::Result; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_templates::ChatTemplates; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::error::chat_templates_error::ChatTemplatesError; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::model::llama_chat_template::LlamaChatTemplate; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::ingested_generation::ingested_generation; +use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const REASONING: &str = "The user wants the weather in Paris."; +const R1_WEATHER_TOOL_CALL: &str = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\n```json\n{\"location\": \"Paris\"}```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; +const END_OF_GENERATION_TOKENS: u64 = 1; + +fn llama_cpp_deepseek_r1_template() -> Result { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../llama-cpp-bindings-sys/llama.cpp/models/templates/llama-cpp-deepseek-r1.jinja"); + + Ok(LlamaChatTemplate::new(&fs::read_to_string(path)?)?) +} + +fn weather_session<'model>( + model: &'model LlamaModel, + templates: &ChatTemplates, +) -> Result> { + Ok(ChatSession::new( + model, + templates, + &[LlamaChatMessage::new( + "user".to_owned(), + "What is the weather in Paris?".to_owned(), + )?], + GET_WEATHER_TOOLS_JSON, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?) +} + +fn token_count(model: &LlamaModel, text: &str) -> Result { + Ok(u64::try_from( + model + .str_to_token(text, AddBos::Never, ParseSpecialTokens::Always)? + .len(), + )?) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 1024, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_served_with_the_llama_cpp_template_parses_its_tool_call( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let templates = + ChatTemplates::with_override(fixture.model, &llama_cpp_deepseek_r1_template()?)?; + let session = weather_session(fixture.model, &templates)?; + + assert_eq!( + recognized_answer( + &session, + &format!("{REASONING}{R1_WEATHER_TOOL_CALL}"), + ChatOutputCompleteness::Finished + ), + ParsedChatMessage::new( + String::new(), + "\nThe user wants the weather in Paris.".to_owned(), + vec![paris_weather_tool_call()] + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 1024, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_served_with_the_llama_cpp_template_streams_its_tool_call_tokens_as_a_tool_call( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let templates = + ChatTemplates::with_override(fixture.model, &llama_cpp_deepseek_r1_template()?)?; + let session = weather_session(fixture.model, &templates)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + + ingested_generation( + &mut classifier, + fixture.model, + &format!("{REASONING}{R1_WEATHER_TOOL_CALL}"), + ); + + assert_eq!( + [ + classifier.usage().reasoning_tokens, + classifier.usage().content_tokens, + classifier.usage().tool_call_tokens, + ], + [ + token_count(fixture.model, REASONING)?, + 0, + token_count(fixture.model, R1_WEATHER_TOOL_CALL)? + END_OF_GENERATION_TOKENS, + ] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 1024, + n_batch = 128, + n_ubatch = 64, +)] +fn an_empty_template_override_is_refused(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + ChatTemplates::with_override(fixture.model, &LlamaChatTemplate::new("")?).err(), + Some(ChatTemplatesError::EmptyTemplateOverride) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs new file mode 100644 index 000000000..5e0688f88 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/completion_usage_accounting.rs @@ -0,0 +1,264 @@ +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn raw_prompt_completion_usage_matches_the_classified_tokens( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let backend = fixture.backend; + let mut ctx = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + ) + .with_context(|| "unable to create context")?; + + let prompt = "Hello my name is"; + let max_generated_tokens: i32 = 64; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let tokens_list = model + .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) + .context(format!("failed to tokenize {prompt}"))?; + let prompt_token_count = u64::try_from(tokens_list.len())?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens_list, 0, false)?; + + ctx.decode(&batch) + .with_context(|| "llama_decode() failed")?; + + classifier.commit_prompt_tokens(); + + let mut sampler = LlamaSampler::chain([LlamaSampler::dist(1234)?, LlamaSampler::greedy()?])?; + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut ctx, + batch: &mut batch, + initial_position, + max_generated_tokens, + } + .run(); + let total_observed = + outcome.observed_content + outcome.observed_reasoning + outcome.observed_undeterminable; + + assert!( + !outcome.generated.text().is_empty(), + "model should generate at least one token" + ); + assert_eq!( + outcome.observed_tool_call, 0, + "raw prompt without tool-call markers must not produce ToolCall tokens; \ + outcome={outcome:?}" + ); + assert!( + total_observed > 0, + "model must produce at least one classified token; outcome={outcome:?}" + ); + + let usage = classifier.into_usage(); + assert_eq!( + usage.prompt_tokens, prompt_token_count, + "prompt_tokens must equal the tokenizer's prompt length" + ); + assert_eq!( + usage.content_tokens, outcome.observed_content, + "content_tokens must equal observed Content variants" + ); + assert_eq!( + usage.reasoning_tokens, outcome.observed_reasoning, + "reasoning_tokens must equal observed Reasoning variants" + ); + assert_eq!( + usage.undeterminable_tokens, outcome.observed_undeterminable, + "undeterminable_tokens must equal observed Undeterminable variants" + ); + assert_eq!( + usage.tool_call_tokens, outcome.observed_tool_call, + "tool_call_tokens must equal observed ToolCall variants" + ); + assert_eq!( + usage.completion_tokens(), + total_observed, + "completion_tokens must equal Content + Reasoning + Undeterminable" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn chat_completion_usage_matches_the_classified_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let backend = fixture.backend; + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let messages = [LlamaChatMessage::new( + "user".to_string(), + "Hello! How are you?".to_string(), + )?]; + let session = ChatSession::new( + model, + model.chat_templates()?, + &messages, + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let tokens = session.tokenize_prompt()?; + let prompt_token_count = u64::try_from(tokens.len())?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; + + context.decode(&batch)?; + + classifier.commit_prompt_tokens(); + + let mut sampler = LlamaSampler::greedy()?; + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position, + max_generated_tokens: 512, + } + .run(); + + assert!( + !outcome.generated.text().is_empty(), + "model should generate at least one token" + ); + let total_observed = + outcome.observed_content + outcome.observed_reasoning + outcome.observed_undeterminable; + assert!( + total_observed > 0, + "model must produce at least one classified token; outcome={outcome:?}" + ); + assert_eq!( + outcome.observed_tool_call, 0, + "chat without tool definitions must not produce ToolCall tokens; outcome={outcome:?}" + ); + + let usage = classifier.into_usage(); + + assert_eq!( + usage.prompt_tokens, prompt_token_count, + "prompt_tokens must equal the tokenizer's prompt length" + ); + assert_eq!( + usage.content_tokens, outcome.observed_content, + "content_tokens must equal observed Content variants" + ); + assert_eq!( + usage.reasoning_tokens, outcome.observed_reasoning, + "reasoning_tokens must equal observed Reasoning variants" + ); + assert_eq!( + usage.undeterminable_tokens, outcome.observed_undeterminable, + "undeterminable_tokens must equal observed Undeterminable variants" + ); + assert_eq!( + usage.completion_tokens(), + total_observed, + "completion_tokens must equal Content + Reasoning + Undeterminable" + ); + assert_eq!( + usage.tool_call_tokens, outcome.observed_tool_call, + "tool_call_tokens must equal observed ToolCall variants" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/context_after_decoding_text.rs b/llama-cpp-bindings-tests/tests/context_after_decoding_text.rs deleted file mode 100644 index d5c63d40e..000000000 --- a/llama-cpp-bindings-tests/tests/context_after_decoding_text.rs +++ /dev/null @@ -1,32 +0,0 @@ -use anyhow::Result; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::params::LlamaPoolingType; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; - -const TEXT: &str = "The parcel arrived on the last working day of the week."; - -pub fn context_after_decoding_text<'fixture>( - fixture: &'fixture LlamaFixture<'_>, - pooling_type: LlamaPoolingType, -) -> Result> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(pooling_type), - )?; - let tokens = fixture - .model - .str_to_token(TEXT, AddBos::Always, ParseSpecialTokens::Never)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - Ok(context) -} diff --git a/llama-cpp-bindings-tests/tests/context_creation_limits.rs b/llama-cpp-bindings-tests/tests/context_creation_limits.rs new file mode 100644 index 000000000..c4ad0c22a --- /dev/null +++ b/llama-cpp-bindings-tests/tests/context_creation_limits.rs @@ -0,0 +1,167 @@ +use std::num::NonZeroU32; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::llama_context_load_error::LlamaContextLoadError; +use llama_cpp_bindings::error::unknown_attention_type::UnknownAttentionType; +use llama_cpp_bindings::error::unknown_pooling_type::UnknownPoolingType; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4294967295, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4294967295, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4294967295, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4294967295, + n_batch = 128, + n_ubatch = 64, +)] +fn a_context_size_overflowing_llama_cpp_padding_is_unconstructible(fixture: &LlamaFixture<'_>) { + assert_eq!( + fixture.build_context().err(), + Some(LlamaContextLoadError::Unconstructible) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 1, + n_ubatch = 1, +)] +fn new_context_rejects_more_sequences_than_one_batch_can_output(fixture: &LlamaFixture<'_>) { + let context_load = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_n_seq_max(2), + ); + + assert_eq!( + context_load.err(), + Some(LlamaContextLoadError::SequencesExceedOutputCapacity { + n_seq_max: 2, + output_capacity: 1, + }) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 2, + n_ubatch = 2, +)] +fn new_context_rejects_more_sequences_than_a_causal_context_can_output(fixture: &LlamaFixture<'_>) { + let context_load = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_n_ctx(NonZeroU32::new(1)) + .with_n_seq_max(2), + ); + + assert_eq!( + context_load.err(), + Some(LlamaContextLoadError::SequencesExceedOutputCapacity { + n_seq_max: 2, + output_capacity: 1, + }) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn creating_a_context_reports_a_batch_llama_cpp_cannot_allocate(fixture: &LlamaFixture<'_>) { + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_INIT_THROWS); + + assert_eq!( + LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + ) + .err(), + Some(LlamaContextLoadError::Reported { + message: injected_fault_message(), + }) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_context_whose_pooling_type_llama_cpp_does_not_define_is_refused(fixture: &LlamaFixture<'_>) { + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_POOLING_TYPE_UNKNOWN); + + assert_eq!( + fixture.build_context().err(), + Some(LlamaContextLoadError::UnknownPoolingType( + UnknownPoolingType(llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_RANK + 1) + )) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn context_params_with_an_attention_type_llama_cpp_does_not_define_are_refused( + fixture: &LlamaFixture<'_>, +) { + let mut params = (*fixture.context_params).into_llama_context_params(); + params.context_params.attention_type = 99; + + assert_eq!( + LlamaContext::from_model(fixture.model, fixture.backend, params).err(), + Some(LlamaContextLoadError::UnknownAttentionType( + UnknownAttentionType(99) + )) + ); +} diff --git a/llama-cpp-bindings-tests/tests/context_dimensions.rs b/llama-cpp-bindings-tests/tests/context_dimensions.rs new file mode 100644 index 000000000..0d84b760c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/context_dimensions.rs @@ -0,0 +1,77 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn context_reports_positive_context_and_batch_sizes(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert!(context.n_ctx() > 0); + assert!(context.n_batch() > 0); + assert!(context.n_ubatch() > 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 512, + n_seq_max = 4, +)] +fn n_ctx_seq_splits_the_context_between_sequences(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + assert_eq!(context.n_ctx(), 2048); + assert_eq!(context.n_ctx_seq(), 512); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/context_state.rs b/llama-cpp-bindings-tests/tests/context_state.rs deleted file mode 100644 index 110738792..000000000 --- a/llama-cpp-bindings-tests/tests/context_state.rs +++ /dev/null @@ -1,2844 +0,0 @@ -use std::num::NonZeroU8; -use std::num::NonZeroU32; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; - -use anyhow::Result; -use anyhow::anyhow; -use llama_cpp_bindings::DecodeError; -use llama_cpp_bindings::EmbeddingsError; -use llama_cpp_bindings::LlamaContextLoadError; -use llama_cpp_bindings::LogitsError; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::params::LlamaAttentionType; -use llama_cpp_bindings::context::params::LlamaPoolingType; -use llama_cpp_bindings::error::ClearKvCacheSeqError; -use llama_cpp_bindings::error::CopyKvCacheSeqError; -use llama_cpp_bindings::error::KvCacheSeqAddError; -use llama_cpp_bindings::error::KvCacheSeqDivError; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings::token::data::LlamaTokenData; -use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; -use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; -use llama_cpp_bindings_tests::prime_kv_cache_with::prime_kv_cache_with; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::download_model::download_model; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn new_context_returns_valid_context(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert!(context.n_ctx() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4294967295, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4294967295, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4294967295, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4294967295, - n_batch = 128, - n_ubatch = 64, -)] -fn new_context_with_huge_ctx_returns_null_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture.build_context(); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 1, - n_ubatch = 1, -)] -fn new_context_rejects_more_sequences_than_one_batch_can_output( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context_load = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_n_seq_max(2), - ); - - assert_eq!( - context_load.err(), - Some(LlamaContextLoadError::SequencesExceedOutputCapacity { - n_seq_max: 2, - output_capacity: 1, - }) - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 2, - n_ubatch = 2, -)] -fn new_context_rejects_more_sequences_than_a_causal_context_can_output( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context_load = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_n_ctx(NonZeroU32::new(1)) - .with_n_seq_max(2), - ); - - assert_eq!( - context_load.err(), - Some(LlamaContextLoadError::SequencesExceedOutputCapacity { - n_seq_max: 2, - output_capacity: 1, - }) - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn context_creation_and_properties(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert!(context.n_ctx() > 0); - assert!(context.n_batch() > 0); - assert!(context.n_ubatch() > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 512, - n_seq_max = 4, -)] -fn n_ctx_seq_splits_the_context_between_sequences(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - assert_eq!(context.n_ctx(), 2048); - assert_eq!(context.n_ctx_seq(), 512); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn decode_and_get_logits(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - - let decode_result = context.decode(&mut batch); - assert!(decode_result.is_ok()); - - let logits = context.get_logits()?; - assert_ne!(logits, Vec::::new()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn timings_work(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.reset_timings(); - let timings = context.timings(); - assert!(timings.t_start_ms() >= 0.0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn token_data_array_has_entries_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let token_data_array = context.token_data_array()?; - - assert_ne!(token_data_array.data, Vec::::new()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn get_logits_ith_returns_valid_slice(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let logits = context.get_logits_ith(last_index)?; - - assert_eq!(logits.len(), usize::try_from(fixture.model.n_vocab())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn token_data_array_ith_returns_valid_data(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let token_data_array = context.token_data_array_ith(last_index)?; - - assert_eq!( - token_data_array.data.len(), - usize::try_from(fixture.model.n_vocab())? - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn embeddings_ith_returns_error_when_embeddings_disabled(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!(context.embeddings_ith(0), Err(EmbeddingsError::NotEnabled)); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn embeddings_seq_ith_returns_error_when_embeddings_disabled( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!( - context.embeddings_seq_ith(0), - Err(EmbeddingsError::NotEnabled) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn candidates_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let count = context.candidates()?.count(); - - assert_eq!(count, usize::try_from(fixture.model.n_vocab())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn debug_format_contains_struct_name(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let debug_output = format!("{context:?}"); - - assert!(debug_output.contains("LlamaContext")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn candidates_ith_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let count = context.candidates_ith(last_index)?.count(); - - assert_eq!(count, usize::try_from(fixture.model.n_vocab())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace( - "Qwen/Qwen2.5-7B-Instruct-GGUF", - "qwen2.5-7b-instruct-q2_k.gguf" - ), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn set_lora_adapters_applies_a_real_adapter(fixture: &LlamaFixture<'_>) -> Result<()> { - let adapter_path = download_model( - "ggml-org/LoRA-Deepthink-Reasoning-Qwen2.5-7B-Instruct-Q8_0-GGUF", - "Deepthink-Reasoning-Adapter-q8_0.gguf", - )?; - let adapter = fixture.model.lora_adapter_init(&adapter_path)?; - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!( - context.set_lora_adapters(&[LoraAdapterScale { - adapter: &adapter, - scale: 1.0, - }]), - Ok(()) - ); - - assert_eq!(context.set_lora_adapters(&[]), Ok(())); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn set_lora_adapters_accepts_an_empty_adapter_set(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let result = context.set_lora_adapters(&[]); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn encode_on_non_encoder_model_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.encode(&mut batch); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn embeddings_seq_ith_returns_null_embedding_error_for_invalid_seq( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::Last), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - assert_eq!( - context.embeddings_seq_ith(999), - Err(EmbeddingsError::SequenceEmbeddingUnavailable { - sequence_index: 999 - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn decode_empty_batch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - let result = context.decode(&mut batch); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn set_abort_flag_aborts_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let abort_flag = Arc::new(AtomicBool::new(true)); - context.set_abort_flag(abort_flag); - - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.decode(&mut batch); - - assert_eq!(result, Err(DecodeError::Aborted)); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn set_abort_flag_false_allows_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let abort_flag = Arc::new(AtomicBool::new(false)); - context.set_abort_flag(abort_flag); - - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.decode(&mut batch); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn clear_abort_callback_allows_decode_with_flag_true(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let abort_flag = Arc::new(AtomicBool::new(true)); - context.set_abort_flag(abort_flag); - context.clear_abort_callback(); - - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.decode(&mut batch); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn synchronize_completes_without_panic(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.synchronize(); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn detach_threadpool_completes_without_panic(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.detach_threadpool(); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn get_logits_ith_returns_token_not_initialized_for_unknown_index( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let result = context.get_logits_ith(7); - - assert_eq!(result, Err(LogitsError::TokenNotInitialized(7))); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 64, - n_batch = 2048, - n_ubatch = 512, -)] -fn get_logits_ith_returns_token_index_exceeds_context_for_huge_index( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let huge_index = i32::try_from(context.n_ctx())?; - context.mark_logits_initialized(huge_index); - let result = context.get_logits_ith(huge_index); - - assert_eq!( - result, - Err(LogitsError::TokenIndexExceedsContext { - token_index: u32::try_from(huge_index)?, - context_size: context.n_ctx(), - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - context.clear_kv_cache(); - assert_eq!(context.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_pos_max_is_non_negative_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - assert!(context.kv_cache_seq_pos_max(0)? >= 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 256, - n_ubatch = 64, -)] -fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - assert!( - prime_kv_cache_with( - fixture, - &mut context, - "Hello\0world", - BatchCapacity::try_from(512)? - ) - .is_err(), - "an interior null byte must surface a tokenization error" - ); - assert!( - prime_kv_cache_with( - fixture, - &mut context, - &"word ".repeat(64), - BatchCapacity::try_from(4)? - ) - .is_err(), - "more tokens than the batch capacity must surface an add-sequence error" - ); - - let filler = "word ".repeat(40); - let filler_batch_capacity = BatchCapacity::try_from(256)?; - let mut decode_result = - prime_kv_cache_with(fixture, &mut context, &filler, filler_batch_capacity); - let mut attempts = 0; - while decode_result.is_ok() && attempts < 16 { - decode_result = prime_kv_cache_with(fixture, &mut context, &filler, filler_batch_capacity); - attempts += 1; - } - assert!( - decode_result.is_err(), - "filling the context past its window must surface a decode error" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_seq_with_range(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - assert_eq!( - context.clear_kv_cache_seq(Some(0), Some(0), Some(1)), - Ok(()) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_kv_cache_seq_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - assert_eq!(context.copy_kv_cache_seq(0, 1, None, None), Ok(())); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - let result = context.kv_cache_seq_add(0, Some(0), None, 1); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqAddError::IncompatibleRopeType - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - let result = context.kv_cache_seq_div(0, Some(0), None, divisor); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqDivError::IncompatibleRopeType - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - context.kv_cache_seq_keep(0); - - assert!(context.kv_cache_seq_pos_max(0)? >= 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_kv_cache_seq_with_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - assert_eq!(context.copy_kv_cache_seq(0, 2, Some(0), Some(1)), Ok(())); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - let result = context.kv_cache_seq_pos_max(999); - - assert_eq!( - result, - Err(llama_cpp_bindings::KvCacheSeqPosMaxError::SequenceIdOutOfRange { seq_id: 999 }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.copy_kv_cache_seq(0, 1, Some(u32::MAX), None); - - assert_eq!( - result.unwrap_err(), - CopyKvCacheSeqError::P0TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.copy_kv_cache_seq(0, 1, Some(0), Some(u32::MAX)); - - assert_eq!( - result.unwrap_err(), - CopyKvCacheSeqError::P1TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_seq_rejects_sequence_id_exceeding_i32_max( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.clear_kv_cache_seq(Some(u32::MAX), None, None); - - assert_eq!( - result.unwrap_err(), - ClearKvCacheSeqError::SeqIdTooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.clear_kv_cache_seq(Some(0), Some(u32::MAX), None); - - assert_eq!( - result.unwrap_err(), - ClearKvCacheSeqError::P0TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.clear_kv_cache_seq(Some(0), Some(0), Some(u32::MAX)); - - assert_eq!( - result.unwrap_err(), - ClearKvCacheSeqError::P1TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.kv_cache_seq_add(0, Some(u32::MAX), None, 1); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqAddError::P0TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.kv_cache_seq_add(0, Some(0), Some(u32::MAX), 1); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqAddError::P1TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - let result = context.kv_cache_seq_div(0, Some(u32::MAX), None, divisor); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqDivError::P0TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - let result = context.kv_cache_seq_div(0, Some(0), Some(u32::MAX), divisor); - - assert_eq!( - result.unwrap_err(), - KvCacheSeqDivError::P1TooLarge( - i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") - ) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn save_and_load_session_file(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let session_path = std::env::temp_dir().join("llama_test_session.bin"); - context.state_save_file(&session_path, &tokens)?; - - let loaded_tokens = context.state_load_file(&session_path, 512)?; - assert_eq!(loaded_tokens, tokens); - - std::fs::remove_file(&session_path)?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn get_state_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - assert!(context.get_state_size() > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let session_path = std::env::temp_dir().join("llama_test_seq_state.bin"); - let bytes_written = context.state_seq_save_file(&session_path, 0, &tokens)?; - assert!(bytes_written > 0); - - let loaded = context.state_seq_load_file(&session_path, 0, 512)?; - - assert_eq!(loaded.tokens, tokens); - assert_eq!( - loaded.bytes_read, bytes_written, - "loading must consume exactly the bytes the save produced" - ); - - std::fs::remove_file(&session_path)?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn set_state_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let mut state_data = vec![0u8; context.get_state_size()]; - let bytes_copied = unsafe { context.copy_state_data(&mut state_data) }?; - state_data.truncate(bytes_copied / 2); - - assert_eq!( - unsafe { context.set_state_data(&state_data) }, - Err( - llama_cpp_bindings::context::state_data_error::StateDataError::NothingRestored { - provided_bytes: state_data.len(), - } - ), - "a truncated snapshot restores nothing, and that must not be reported as success" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let state_size = context.get_state_size(); - let mut state_data = vec![0u8; state_size]; - let bytes_copied = unsafe { context.copy_state_data(&mut state_data) }?; - assert!(bytes_copied > 0); - - let bytes_read = unsafe { context.set_state_data(&state_data) }?; - assert_eq!( - bytes_read, bytes_copied, - "restoring the state must consume exactly the bytes the snapshot produced" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_nonexistent_file_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.state_load_file("/nonexistent/session.bin", 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_nonexistent_file_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let result = context.state_seq_load_file("/nonexistent/seq_state.bin", 0, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_save_file_to_invalid_directory_returns_failed_to_save( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = fixture.build_context()?; - - let result = context.state_save_file("/nonexistent_dir/session.bin", &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_save_file_to_invalid_directory_returns_failed_to_save( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = fixture.build_context()?; - - let result = context.state_seq_save_file("/nonexistent_dir/seq_state.bin", 0, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let session_path = std::env::temp_dir().join("llama_test_session_zero_max.bin"); - context.state_save_file(&session_path, &tokens)?; - - let result = context.state_load_file(&session_path, 0); - - assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_zero_max_tokens_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let session_path = std::env::temp_dir().join("llama_test_seq_state_zero_max.bin"); - context.state_seq_save_file(&session_path, 0, &tokens)?; - - let result = context.state_seq_load_file(&session_path, 0, 0); - - assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_insufficient_max_tokens_returns_length_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = fixture.model.str_to_token( - "Hello world this is a longer string for more tokens", - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let session_path = std::env::temp_dir().join("llama_test_session_insuf.bin"); - context.state_save_file(&session_path, &tokens)?; - - let result = context.state_load_file(&session_path, 1); - - assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let tokens = fixture.model.str_to_token( - "Hello world this is a longer string for more tokens", - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let session_path = std::env::temp_dir().join("llama_test_seq_state_insuf.bin"); - context.state_seq_save_file(&session_path, 0, &tokens)?; - - let result = context.state_seq_load_file(&session_path, 0, 1); - - assert!(result.is_err()); - let _ = std::fs::remove_file(&session_path); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let context = fixture.build_context()?; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_save_file(non_utf8_path, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let mut context = fixture.build_context()?; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_load_file(non_utf8_path, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let context = fixture.build_context()?; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_seq_save_file(non_utf8_path, 0, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let mut context = fixture.build_context()?; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); - let result = context.state_seq_load_file(non_utf8_path, 0, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); - let result = context.state_save_file(path_with_null, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); - let result = context.state_load_file(path_with_null, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_save_file_with_null_byte_in_path_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = fixture.build_context()?; - - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); - let result = context.state_seq_save_file(path_with_null, 0, &[]); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_load_file_with_null_byte_in_path_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - let path_with_null = std::path::Path::new("/tmp/foo\0bar.bin"); - let result = context.state_seq_load_file(path_with_null, 0, 512); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_get_size_ext_returns_size_for_decoded_sequence( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; - - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let flags = LlamaStateSeqFlags::empty(); - let size = context.state_seq_get_size_ext(0, &flags); - - assert!(size > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn state_seq_get_data_ext_and_set_data_ext_round_trip(fixture: &LlamaFixture<'_>) -> Result<()> { - use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; - - let mut context = fixture.build_context()?; - - let tokens = - fixture - .model - .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let flags = LlamaStateSeqFlags::empty(); - let size = context.state_seq_get_size_ext(0, &flags); - let mut buffer = vec![0u8; size]; - let bytes_written = unsafe { context.state_seq_get_data_ext(&mut buffer, 0, &flags) }?; - - assert!(bytes_written > 0); - - let bytes_read = unsafe { context.state_seq_set_data_ext(&buffer, 0, &flags) }?; - - assert_eq!( - bytes_read, bytes_written, - "restoring the sequence state must consume exactly the bytes the snapshot produced" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_refilled_candidate_array_matches_a_freshly_built_one( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let prompt_tokens = model.str_to_token("Hello", AddBos::Never, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(64)?, 1); - - batch.add_sequence(&prompt_tokens, 0, false)?; - context.decode(&mut batch)?; - - let last_index = batch.n_tokens() - 1; - let mut reused_candidates = LlamaTokenDataArray::new( - vec![LlamaTokenData::new(LlamaToken::new(7), 1.0, 0.5)], - true, - ); - reused_candidates.selected = Some(0); - - context.fill_token_data_array_ith(last_index, &mut reused_candidates)?; - - assert_eq!(reused_candidates, context.token_data_array_ith(last_index)?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, -)] -fn decoding_more_tokens_than_the_micro_batch_with_non_causal_attention_returns_an_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_attention_type(LlamaAttentionType::NonCausal), - )?; - let tokens = fixture.model.str_to_token( - &"hello ".repeat(100), - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - let n_tokens = batch.n_tokens(); - - assert_eq!( - context.decode(&mut batch), - Err(DecodeError::BatchExceedsMicroBatch { - n_tokens, - n_ubatch: context.n_ubatch(), - }) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/debug_formatting.rs b/llama-cpp-bindings-tests/tests/debug_formatting.rs new file mode 100644 index 000000000..fa7698519 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/debug_formatting.rs @@ -0,0 +1,41 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn context_debug_format_contains_struct_name(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let debug_output = format!("{context:?}"); + + assert!(debug_output.contains("LlamaContext")); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128 +)] +fn model_debug_format_includes_struct_name_and_model_field(fixture: &LlamaFixture<'_>) { + let formatted = format!("{:?}", fixture.model); + + assert!(formatted.contains("LlamaModel")); + assert!(formatted.contains("model")); +} diff --git a/llama-cpp-bindings-tests/tests/decode_abort_control.rs b/llama-cpp-bindings-tests/tests/decode_abort_control.rs new file mode 100644 index 000000000..3d8fb541c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/decode_abort_control.rs @@ -0,0 +1,100 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::decode_error::DecodeError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn set_abort_flag_aborts_decode(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let abort_flag = Arc::new(AtomicBool::new(true)); + context.set_abort_flag(abort_flag); + + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let result = context.decode(&batch); + + assert_eq!(result, Err(DecodeError::Aborted)); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn set_abort_flag_false_allows_decode(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let abort_flag = Arc::new(AtomicBool::new(false)); + context.set_abort_flag(abort_flag); + + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + assert_eq!(context.decode(&batch), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn clear_abort_callback_allows_decode_with_flag_true(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let abort_flag = Arc::new(AtomicBool::new(true)); + context.set_abort_flag(abort_flag); + context.clear_abort_callback(); + + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + assert_eq!(context.decode(&batch), Ok(())); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs new file mode 100644 index 000000000..4e96409df --- /dev/null +++ b/llama-cpp-bindings-tests/tests/decode_batch_rejection.rs @@ -0,0 +1,137 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_attention_type::LlamaAttentionType; +use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; +use llama_cpp_bindings::error::decode_error::DecodeError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn decoding_an_empty_batch_is_refused_as_invalid(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let batch = LlamaBatch::new(ContextLimits::of(&context)); + + assert_eq!(context.decode(&batch), Err(DecodeError::BatchInvalid)); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, +)] +fn decoding_more_tokens_than_the_micro_batch_with_non_causal_attention_is_refused( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_attention_type(LlamaAttentionType::NonCausal), + )?; + let tokens = fixture.model.str_to_token( + &"hello ".repeat(100), + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let n_tokens = batch.n_tokens(); + + assert_eq!( + context.decode(&batch), + Err(DecodeError::BatchExceedsMicroBatch { + n_tokens, + n_ubatch: context.n_ubatch(), + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_is_refused( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture.model.str_to_token( + &"hello ".repeat(100), + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let n_tokens = batch.n_tokens(); + + assert_eq!( + context.decode(&batch), + Err(DecodeError::BatchExceedsMicroBatch { + n_tokens, + n_ubatch: context.n_ubatch(), + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn decoding_reports_an_exception_thrown_while_transferring_the_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS, + ); + + assert_eq!( + context.decode(&batch), + Err(DecodeError::BatchTransfer(BatchTransferError::Reported { + message: injected_fault_message(), + })) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/detokenization.rs b/llama-cpp-bindings-tests/tests/detokenization.rs new file mode 100644 index 000000000..69c0b306c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/detokenization.rs @@ -0,0 +1,368 @@ +use std::ffi::c_int; +use std::num::NonZeroU16; + +use anyhow::Context as _; +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_tests::single_token::single_token; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const PIECE_PROBE_BYTES: usize = 32; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_bytes_returns_the_bytes_of_the_token_piece( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + + assert_eq!( + model.token_to_piece_bytes(tokens[0], PIECE_PROBE_BYTES, false, None)?, + b"hello" + ); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_returns_a_piece_longer_than_its_probe_buffer_whole( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let long_token = (0..model.n_vocab()) + .map(LlamaToken::new) + .find(|token| { + matches!( + model.token_to_piece_bytes(*token, PIECE_PROBE_BYTES, false, None), + Err(TokenToStringError::InsufficientBufferSpace(_)) + ) + }) + .context("the vocabulary has a piece longer than the probe buffer")?; + let Err(TokenToStringError::InsufficientBufferSpace(negated_piece_size)) = + model.token_to_piece_bytes(long_token, PIECE_PROBE_BYTES, false, None) + else { + bail!("the probe buffer must be too small for the long piece"); + }; + let long_piece = model.token_to_piece_bytes( + long_token, + usize::try_from(negated_piece_size.unsigned_abs())?, + false, + None, + )?; + let mut decoder = encoding_rs::UTF_8.new_decoder(); + + assert_eq!( + model.token_to_piece( + &SampledToken::Content(long_token), + &mut decoder, + false, + None + )?, + String::from_utf8(long_piece)? + ); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_bytes_with_a_too_small_buffer_reports_the_size_the_piece_needs( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + + assert_eq!( + model.token_to_piece_bytes(tokens[0], 1, false, None), + Err(TokenToStringError::InsufficientBufferSpace( + -c_int::try_from("hello".len())? + )) + ); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_strips_the_requested_count_of_leading_spaces( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let tokens = model.str_to_token(" hello", AddBos::Never, ParseSpecialTokens::Always)?; + + assert_eq!( + model.token_to_piece( + &SampledToken::Content(tokens[0]), + &mut decoder, + false, + NonZeroU16::new(1), + )?, + "hello" + ); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_decodes_a_token_alike_in_every_non_content_section(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let hi_token = single_token(model, "hi"); + + assert_eq!( + [ + SampledToken::Reasoning(hi_token), + SampledToken::ToolCall(hi_token), + SampledToken::Undeterminable(hi_token), + ] + .map(|sampled_token| { + model.token_to_piece( + &sampled_token, + &mut encoding_rs::UTF_8.new_decoder(), + true, + None, + ) + }), + [ + Ok("hi".to_owned()), + Ok("hi".to_owned()), + Ok("hi".to_owned()) + ] + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_bytes_refuses_a_buffer_beyond_a_c_int_before_allocating( + fixture: &LlamaFixture<'_>, +) { + assert_eq!( + fixture + .model + .token_to_piece_bytes(LlamaToken::new(0), usize::MAX, false, None) + .err(), + i32::try_from(usize::MAX) + .err() + .map(TokenToStringError::IntConversionError) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_replaces_a_lone_continuation_byte_with_the_replacement_character( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let continuation_byte_token = (0..model.n_vocab()) + .map(LlamaToken::new) + .find(|token| { + model + .token_to_piece_bytes(*token, 8, true, None) + .is_ok_and(|bytes| bytes == [0x80]) + }) + .context("the byte-level vocabulary has a token for the byte 0x80")?; + let mut decoder = encoding_rs::UTF_8.new_decoder(); + + assert_eq!( + model.token_to_piece( + &SampledToken::Content(continuation_byte_token), + &mut decoder, + true, + None + )?, + "\u{FFFD}" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_control_token_decoded_without_special_rendering_is_an_empty_piece(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let control_token = single_token(model, "<|im_start|>"); + + assert_eq!( + model.token_to_piece( + &SampledToken::Content(control_token), + &mut encoding_rs::UTF_8.new_decoder(), + false, + None + ), + Ok(String::new()) + ); +} diff --git a/llama-cpp-bindings-tests/tests/dry_sampler.rs b/llama-cpp-bindings-tests/tests/dry_sampler.rs new file mode 100644 index 000000000..8477f9e2e --- /dev/null +++ b/llama-cpp-bindings-tests/tests/dry_sampler.rs @@ -0,0 +1,47 @@ +use std::ffi::CString; + +use anyhow::Result; + +use llama_cpp_bindings::error::sampling_error::SamplingError; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn building_a_dry_sampler_with_sequence_breakers_succeeds( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let breakers: Vec<&[u8]> = vec![b"\n", b"\t"]; + + LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers)?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn dry_sampler_refuses_a_sequence_breaker_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + let breakers: Vec<&[u8]> = vec![b"hello\0world"]; + let result = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, breakers); + + assert_eq!( + result.err(), + Some(SamplingError::DrySequenceBreakerContainsNul { + index: 0, + source: CString::new(&b"hello\0world"[..]).unwrap_err(), + }) + ); +} diff --git a/llama-cpp-bindings-tests/tests/embedding_availability.rs b/llama-cpp-bindings-tests/tests/embedding_availability.rs index 0bc533cfc..0a2bf16ee 100644 --- a/llama-cpp-bindings-tests/tests/embedding_availability.rs +++ b/llama-cpp-bindings-tests/tests/embedding_availability.rs @@ -1,42 +1,38 @@ -use anyhow::Result; -use llama_cpp_bindings::EmbeddingsError; -use llama_cpp_bindings::context::params::LlamaPoolingType; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -use crate::context_after_decoding_text::context_after_decoding_text; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::error::embeddings_error::EmbeddingsError; +use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, n_ubatch = 512, embeddings = true, )] -fn sequence_embeddings_need_a_pooling_context(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::None)?; +fn sequence_embeddings_need_a_pooling_context(fixture: &LlamaFixture<'_>) { + let context = context_after_decoding_text(fixture, LlamaPoolingType::None); assert_eq!( context.embeddings_seq_ith(0), Err(EmbeddingsError::NonePoolType) ); - - Ok(()) } #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, n_ubatch = 512, embeddings = true, )] -fn token_embeddings_need_a_context_without_pooling(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; +fn token_embeddings_need_a_context_without_pooling(fixture: &LlamaFixture<'_>) { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); assert_eq!( context.embeddings_ith(0), @@ -44,6 +40,21 @@ fn token_embeddings_need_a_context_without_pooling(fixture: &LlamaFixture<'_>) - pooling_type: LlamaPoolingType::Mean, }) ); +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn disabling_embeddings_makes_token_embeddings_unavailable(fixture: &LlamaFixture<'_>) { + let mut context = context_after_decoding_text(fixture, LlamaPoolingType::None); + + context.set_embeddings(false); - Ok(()) + assert_eq!(context.embeddings_ith(-1), Err(EmbeddingsError::NotEnabled)); } diff --git a/llama-cpp-bindings-tests/tests/embedding_generation.rs b/llama-cpp-bindings-tests/tests/embedding_generation.rs new file mode 100644 index 000000000..cbb1d0222 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/embedding_generation.rs @@ -0,0 +1,267 @@ +use anyhow::Context; +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_threads_batch = 8, + embeddings = true, +)] +fn an_embedding_pass_records_its_prompt_tokens_and_no_completion_tokens( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + + let mut ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + ) + .with_context(|| "unable to create context")?; + + let prompt = "Hello my name is"; + let tokens = model + .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) + .context(format!("failed to tokenize {prompt}"))?; + let prompt_token_count = u64::try_from(tokens.len())?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; + + ctx.decode(&batch) + .with_context(|| "llama_decode() failed")?; + classifier.commit_prompt_tokens(); + + let usage = classifier.into_usage(); + assert_eq!(usage.prompt_tokens, prompt_token_count); + assert_eq!(usage.completion_tokens(), 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn a_sequence_embedding_is_as_wide_as_the_model_embedding( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let embeddings = context.embeddings_seq_ith(0)?; + + assert_eq!(embeddings.len(), usize::try_from(fixture.model.n_embd())?); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_seq_max = 4, + embeddings = true, +)] +fn multi_sequence_embeddings_returns_one_embedding_per_sequence( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let inputs = [ + "alpha is here", + "beta runs fast", + "gamma waits", + "delta jumps", + ]; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + for (sequence_index, text) in inputs.iter().enumerate() { + let tokens = + fixture + .model + .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; + let sequence_id = i32::try_from(sequence_index)?; + + batch.add_sequence(&tokens, sequence_id, true)?; + } + + context.decode(&batch)?; + + let n_embd = usize::try_from(fixture.model.n_embd())?; + let mut collected: Vec> = Vec::with_capacity(inputs.len()); + + for sequence_index in 0..inputs.len() { + let sequence_id = i32::try_from(sequence_index)?; + let embedding = context.embeddings_seq_ith(sequence_id)?; + + assert_eq!( + embedding.len(), + n_embd, + "sequence {sequence_index} embedding length mismatch" + ); + + collected.push(embedding.to_vec()); + } + + for (left_index, left) in collected.iter().enumerate() { + for (right_index, right) in collected.iter().enumerate().skip(left_index + 1) { + assert_ne!( + left, right, + "embedding for sequence {left_index} must differ from sequence {right_index}", + ); + } + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_seq_max = 4, + embeddings = true, +)] +fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let iterations = [ + [ + "This is the first document with enough content to contribute meaningfully to the batch size calculation", + "This is the second document that should be processed in a potentially different batch from the first", + ], + [ + "This is the third document adding more content to ensure the total exceeds the configured chunk limit", + "This is the fourth document which should demonstrate that batching distributes across agent requests", + ], + ]; + + let n_embd = usize::try_from(fixture.model.n_embd())?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let mut collected: Vec> = Vec::new(); + + for iteration_inputs in iterations { + for (sequence_index, text) in iteration_inputs.iter().enumerate() { + let tokens = + fixture + .model + .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; + let sequence_id = i32::try_from(sequence_index)?; + + batch.add_sequence(&tokens, sequence_id, true)?; + } + + context.memory_clear(); + context.decode(&batch)?; + + for sequence_index in 0..iteration_inputs.len() { + let sequence_id = i32::try_from(sequence_index)?; + let embedding = context.embeddings_seq_ith(sequence_id)?; + + assert_eq!( + embedding.len(), + n_embd, + "iteration sequence {sequence_index} embedding length mismatch" + ); + + collected.push(embedding.to_vec()); + } + + batch.clear(); + } + + assert_eq!( + collected.len(), + iterations.iter().flatten().count(), + "expected one embedding per input across every iteration" + ); + + for (left_index, left) in collected.iter().enumerate() { + for (right_index, right) in collected.iter().enumerate().skip(left_index + 1) { + assert_ne!( + left, right, + "embedding {left_index} must differ from embedding {right_index} across reused-batch iterations", + ); + } + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn a_token_embedding_is_as_wide_as_the_model_embedding(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_pooling_type(LlamaPoolingType::None), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let last_index = i32::try_from(tokens.len() - 1)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let embeddings = context.embeddings_ith(last_index)?; + + assert_eq!(embeddings.len(), usize::try_from(fixture.model.n_embd())?); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/embedding_models.rs b/llama-cpp-bindings-tests/tests/embedding_models.rs deleted file mode 100644 index bbaf72c7c..000000000 --- a/llama-cpp-bindings-tests/tests/embedding_models.rs +++ /dev/null @@ -1,833 +0,0 @@ -use std::num::NonZeroU8; -use std::time::Duration; - -use anyhow::Context; -use anyhow::Result; -use anyhow::anyhow; -use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ClearKvCacheSeqError; -use llama_cpp_bindings::CopyKvCacheSeqError; -use llama_cpp_bindings::DecodeError; -use llama_cpp_bindings::EmbeddingsError; -use llama_cpp_bindings::EncodeError; -use llama_cpp_bindings::KvCacheSeqAddError; -use llama_cpp_bindings::KvCacheSeqDivError; -use llama_cpp_bindings::KvCacheSeqPosMaxError; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::params::LlamaPoolingType; -use llama_cpp_bindings::ggml_time_us; -use llama_cpp_bindings::gguf_context::GgufContext; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::model::params::LlamaModelParams; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings_tests::prime_kv_cache::prime_kv_cache; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; -use toktrie::INVALID_TOKEN; -use toktrie::TokenizerEnv as _; - -fn normalize(input: &[f32]) -> Vec { - let magnitude = input - .iter() - .fold(0.0, |accumulator, &value| value.mul_add(value, accumulator)) - .sqrt(); - - input.iter().map(|&value| value / magnitude).collect() -} - -fn cosine_similarity(vec_a: &[f32], vec_b: &[f32]) -> f32 { - vec_a - .iter() - .zip(vec_b.iter()) - .map(|(left, right)| left * right) - .sum::() -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_threads_batch = 8, - embeddings = true, -)] -fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - - let mut ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - ) - .with_context(|| "unable to create context")?; - - let prompt = "Hello my name is"; - let tokens = model - .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) - .with_context(|| format!("failed to tokenize {prompt}"))?; - let prompt_token_count = u64::try_from(tokens.len())?; - - let n_ctx = usize::try_from(ctx.n_ctx())?; - assert!(tokens.len() <= n_ctx, "prompt exceeds context window size"); - - let t_main_start = ggml_time_us(); - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(ctx.n_ctx())?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - ctx.clear_kv_cache(); - ctx.decode(&mut batch) - .with_context(|| "llama_decode() failed")?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let embedding = ctx - .embeddings_seq_ith(0) - .with_context(|| "failed to get embeddings")?; - let normalized = normalize(embedding); - - let t_main_end = ggml_time_us(); - let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); - - eprintln!( - "created embedding with {} dimensions in {:.2} s", - normalized.len(), - duration.as_secs_f32() - ); - - assert!( - !normalized.is_empty(), - "embedding should have at least one dimension" - ); - - let magnitude: f32 = normalized - .iter() - .map(|value| value * value) - .sum::() - .sqrt(); - assert!( - (magnitude - 1.0).abs() < 0.01, - "normalized embedding magnitude should be approximately 1.0, got {magnitude}" - ); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, prompt_token_count); - assert_eq!(usage.completion_tokens(), 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_seq_max = 2, - n_threads_batch = 8, - embeddings = true, -)] -fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - - let query = "What is machine learning?"; - let documents = [ - "Machine learning is a subset of artificial intelligence.", - "The weather today is sunny and warm.", - ]; - - let document_count = documents.len(); - assert_eq!( - u32::try_from(document_count)?, - fixture.context_params.n_seq_max, - "attribute n_seq_max must match the document count this trial expects", - ); - - let mut ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - ) - .with_context(|| "unable to create context")?; - - let prompt_lines: Vec = documents - .iter() - .map(|document| format!("{query}{document}")) - .collect(); - - let tokens_lines_list = prompt_lines - .iter() - .map(|line| model.str_to_token(line, AddBos::Always, ParseSpecialTokens::Always)) - .collect::, _>>() - .with_context(|| "failed to tokenize prompts")?; - - let n_ctx = usize::try_from(ctx.n_ctx())?; - - if tokens_lines_list.iter().any(|tokens| n_ctx < tokens.len()) { - bail!("one of the provided prompts exceeds the size of the context window"); - } - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new( - BatchCapacity::try_from(2048)?, - i32::try_from(document_count)?, - ); - let t_main_start = ggml_time_us(); - - for (sequence_index, tokens) in tokens_lines_list.iter().enumerate() { - classifier.feed_prompt_sequence_to_batch( - &mut batch, - tokens, - i32::try_from(sequence_index)?, - false, - )?; - } - - let total_tokens: usize = tokens_lines_list.iter().map(Vec::len).sum(); - let total_token_count = u64::try_from(total_tokens)?; - - assert_eq!(classifier.pending_prompt_tokens(), total_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - ctx.clear_kv_cache(); - ctx.decode(&mut batch) - .with_context(|| "llama_decode() failed")?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, total_token_count); - - let mut embeddings = Vec::with_capacity(document_count); - - for sequence_index in 0..document_count { - let raw_embedding = ctx - .embeddings_seq_ith(i32::try_from(sequence_index)?) - .with_context(|| "failed to get sequence embeddings")?; - embeddings.push(normalize(raw_embedding)); - } - - let t_main_end = ggml_time_us(); - let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); - - let tokens_per_second = f64::from(u32::try_from(total_tokens)?) / duration.as_secs_f64(); - - eprintln!( - "created embeddings for {total_tokens} tokens in {:.2} s, speed {tokens_per_second:.2} t/s", - duration.as_secs_f64(), - ); - - assert_eq!( - embeddings.len(), - document_count, - "should produce one embedding per document" - ); - - for (index, embedding) in embeddings.iter().enumerate() { - assert!( - !embedding.is_empty(), - "embedding {index} should not be empty" - ); - } - - let similarity = cosine_similarity(&embeddings[0], &embeddings[1]); - eprintln!("cosine similarity between document embeddings: {similarity:.4}"); - - assert!( - similarity.is_finite(), - "cosine similarity should be a finite number" - ); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, total_token_count); - assert_eq!(usage.completion_tokens(), 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn decode_with_embeddings_enabled(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.decode(&mut batch); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn embeddings_seq_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let embeddings = context.embeddings_seq_ith(0)?; - - assert_eq!(embeddings.len(), usize::try_from(fixture.model.n_embd())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_seq_max = 4, - embeddings = true, -)] -fn multi_sequence_embeddings_returns_one_embedding_per_sequence( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let inputs = [ - "alpha is here", - "beta runs fast", - "gamma waits", - "delta jumps", - ]; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(64)?, 4); - - for (sequence_index, text) in inputs.iter().enumerate() { - let tokens = - fixture - .model - .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; - let sequence_id = i32::try_from(sequence_index)?; - - batch.add_sequence(&tokens, sequence_id, true)?; - } - - context.decode(&mut batch)?; - - let n_embd = usize::try_from(fixture.model.n_embd())?; - let mut collected: Vec> = Vec::with_capacity(inputs.len()); - - for sequence_index in 0..inputs.len() { - let sequence_id = i32::try_from(sequence_index)?; - let embedding = context.embeddings_seq_ith(sequence_id)?; - - assert_eq!( - embedding.len(), - n_embd, - "sequence {sequence_index} embedding length mismatch" - ); - - collected.push(embedding.to_vec()); - } - - for (left_index, left) in collected.iter().enumerate() { - for (right_index, right) in collected.iter().enumerate().skip(left_index + 1) { - assert_ne!( - left, right, - "embedding for sequence {left_index} must differ from sequence {right_index}", - ); - } - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_seq_max = 4, - embeddings = true, -)] -fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let iterations = [ - [ - "This is the first document with enough content to contribute meaningfully to the batch size calculation", - "This is the second document that should be processed in a potentially different batch from the first", - ], - [ - "This is the third document adding more content to ensure the total exceeds the configured chunk limit", - "This is the fourth document which should demonstrate that batching distributes across agent requests", - ], - ]; - - let n_embd = usize::try_from(fixture.model.n_embd())?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(64)?, 4); - let mut collected: Vec> = Vec::new(); - - for iteration_inputs in iterations { - for (sequence_index, text) in iteration_inputs.iter().enumerate() { - let tokens = - fixture - .model - .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; - let sequence_id = i32::try_from(sequence_index)?; - - batch.add_sequence(&tokens, sequence_id, true)?; - } - - context.clear_kv_cache(); - context.decode(&mut batch)?; - - for sequence_index in 0..iteration_inputs.len() { - let sequence_id = i32::try_from(sequence_index)?; - let embedding = context.embeddings_seq_ith(sequence_id)?; - - assert_eq!( - embedding.len(), - n_embd, - "iteration sequence {sequence_index} embedding length mismatch" - ); - - collected.push(embedding.to_vec()); - } - - batch.clear(); - } - - assert_eq!( - collected.len(), - iterations.iter().flatten().count(), - "expected one embedding per input across every iteration" - ); - - for (left_index, left) in collected.iter().enumerate() { - for (right_index, right) in collected.iter().enumerate().skip(left_index + 1) { - assert_ne!( - left, right, - "embedding {left_index} must differ from embedding {right_index} across reused-batch iterations", - ); - } - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn embeddings_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::None), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let last_index = i32::try_from(tokens.len() - 1)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let embeddings = context.embeddings_ith(last_index)?; - - assert_eq!(embeddings.len(), usize::try_from(fixture.model.n_embd())?); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn embeddings_ith_returns_null_embedding_error_for_non_embedding_token( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::None), - )?; - - assert_eq!( - context.embeddings_ith(999), - Err(EmbeddingsError::LogitsNotEnabled) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - embeddings = true, -)] -fn encode_succeeds_with_encoder_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - - let result = context.encode(&mut batch); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_add_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - let result = context.kv_cache_seq_add(0, Some(0), None, 1); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn kv_cache_seq_div_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = fixture.build_context()?; - - prime_kv_cache(fixture, &mut context)?; - - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - let result = context.kv_cache_seq_div(0, Some(0), None, divisor); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128 -)] -fn embedding_model_exposes_tool_call_markers(fixture: &LlamaFixture<'_>) -> Result<()> { - let markers = fixture - .model - .tool_call_markers()? - .expect("Qwen3 embedding model must expose tool-call markers"); - - assert_ne!(markers.open, ""); - assert_ne!(markers.close, ""); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128 -)] -fn embedding_model_exposes_streaming_markers(fixture: &LlamaFixture<'_>) -> Result<()> { - let markers = fixture.model.streaming_markers()?; - - assert!(!markers.is_empty()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, - embeddings = true, -)] -fn an_encoder_context_keeps_no_memory(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(!fixture.build_context()?.has_memory()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, - embeddings = true, -)] -fn a_causal_embedding_context_keeps_memory(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.build_context()?.has_memory()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - n_threads_batch = 8, - embeddings = true, -)] -fn kv_cache_operations_respect_an_embedding_context_without_memory( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - - context.clear_kv_cache(); - assert_eq!(context.clear_kv_cache_seq(Some(0), None, None), Ok(())); - context.kv_cache_seq_keep(0); - - assert_eq!( - context.copy_kv_cache_seq(0, 1, None, None), - Err(CopyKvCacheSeqError::MemoryHandleUnavailable) - ); - assert_eq!( - context.kv_cache_seq_add(0, None, None, 1), - Err(KvCacheSeqAddError::MemoryHandleUnavailable) - ); - let divisor = NonZeroU8::new(2).ok_or_else(|| anyhow!("2 is non-zero"))?; - assert_eq!( - context.kv_cache_seq_div(0, None, None, divisor), - Err(KvCacheSeqDivError::MemoryHandleUnavailable) - ); - assert_eq!( - context.kv_cache_seq_pos_max(0), - Err(KvCacheSeqPosMaxError::MemoryHandleUnavailable) - ); - assert!(matches!( - context.clear_kv_cache_seq(Some(u32::MAX), None, None), - Err(ClearKvCacheSeqError::SeqIdTooLarge(_)) - )); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn approximate_tok_env_falls_back_to_eos_when_eot_unavailable( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let LlamaToken(eos) = fixture - .model - .token_eos() - .context("model declares an EOS token")?; - - assert_eq!( - fixture.model.approximate_tok_env()?.tok_trie().eos_token(), - eos.cast_unsigned() - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn approximate_tok_env_of_a_model_without_eot_and_eos_has_no_eos( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model_without_eos_path = std::env::temp_dir().join(format!( - "llama_cpp_bindings_{}_model_without_eos.gguf", - std::process::id() - )); - let mut gguf_context = GgufContext::from_file(fixture.model_path)?; - - gguf_context.remove_key("tokenizer.ggml.eos_token_id")?; - gguf_context.write_to_file(&model_without_eos_path)?; - - let model_without_eos = LlamaModel::load_from_file( - fixture.backend, - &model_without_eos_path, - &LlamaModelParams::default(), - )?; - - std::fs::remove_file(&model_without_eos_path)?; - - assert_eq!( - model_without_eos - .approximate_tok_env()? - .tok_trie() - .eos_token(), - INVALID_TOKEN - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn decoding_more_tokens_than_the_micro_batch_on_an_encoder_returns_an_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - let tokens = fixture.model.str_to_token( - &"hello ".repeat(100), - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - let n_tokens = batch.n_tokens(); - - assert_eq!( - context.decode(&mut batch), - Err(DecodeError::BatchExceedsMicroBatch { - n_tokens, - n_ubatch: context.n_ubatch(), - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn encoding_more_tokens_than_the_micro_batch_returns_an_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = fixture.build_context()?; - let tokens = fixture.model.str_to_token( - &"hello ".repeat(100), - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - let n_tokens = batch.n_tokens(); - - assert_eq!( - context.encode(&mut batch), - Err(EncodeError::BatchExceedsMicroBatch { - n_tokens, - n_ubatch: context.n_ubatch(), - }) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs new file mode 100644 index 000000000..e05d58c8e --- /dev/null +++ b/llama-cpp-bindings-tests/tests/embedding_retrieval_errors.rs @@ -0,0 +1,122 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::error::embeddings_error::EmbeddingsError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn token_embedding_reports_embeddings_not_enabled_on_a_context_without_embeddings( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!(context.embeddings_ith(0), Err(EmbeddingsError::NotEnabled)); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn sequence_embedding_reports_embeddings_not_enabled_on_a_context_without_embeddings( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!( + context.embeddings_seq_ith(0), + Err(EmbeddingsError::NotEnabled) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn sequence_embedding_of_a_sequence_missing_from_the_decoded_batch_is_unavailable( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_pooling_type(LlamaPoolingType::Last), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + assert_eq!( + context.embeddings_seq_ith(999), + Err(EmbeddingsError::SequenceEmbeddingUnavailable { + sequence_index: 999 + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn token_embedding_of_an_index_without_a_decoded_output_names_no_output_row( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_pooling_type(LlamaPoolingType::None), + )?; + + assert_eq!( + context.embeddings_ith(999), + Err(EmbeddingsError::TokenNotAnOutput { token_index: 999 }) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/embedding_row_width.rs b/llama-cpp-bindings-tests/tests/embedding_row_width.rs index af80617a2..cc2650fc1 100644 --- a/llama-cpp-bindings-tests/tests/embedding_row_width.rs +++ b/llama-cpp-bindings-tests/tests/embedding_row_width.rs @@ -1,16 +1,16 @@ use anyhow::Result; -use llama_cpp_bindings::context::params::LlamaPoolingType; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; -use crate::context_after_decoding_text::context_after_decoding_text; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const LFM2_COLBERT_OUTPUT_PROJECTION_WIDTH: usize = 128; const QWEN3_RERANKER_CLASSIFIER_LABELS: usize = 2; #[llama_test( model_source = HuggingFace("LiquidAI/LFM2-ColBERT-350M-GGUF", "LFM2-ColBERT-350M-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -18,7 +18,7 @@ const QWEN3_RERANKER_CLASSIFIER_LABELS: usize = 2; embeddings = true, )] fn token_embeddings_are_as_wide_as_the_output_projection(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::None)?; + let context = context_after_decoding_text(fixture, LlamaPoolingType::None); assert_eq!( context.embeddings_ith(0)?.len(), @@ -30,7 +30,7 @@ fn token_embeddings_are_as_wide_as_the_output_projection(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("LiquidAI/LFM2-ColBERT-350M-GGUF", "LFM2-ColBERT-350M-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -40,7 +40,7 @@ fn token_embeddings_are_as_wide_as_the_output_projection(fixture: &LlamaFixture< fn pooled_sequence_embeddings_are_as_wide_as_the_output_projection( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean)?; + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); assert_eq!( context.embeddings_seq_ith(0)?.len(), @@ -52,7 +52,7 @@ fn pooled_sequence_embeddings_are_as_wide_as_the_output_projection( #[llama_test( model_source = HuggingFace("ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF", "qwen3-reranker-0.6b-q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 512, @@ -62,7 +62,7 @@ fn pooled_sequence_embeddings_are_as_wide_as_the_output_projection( fn rank_pooled_sequence_embeddings_hold_one_score_per_classifier_label( fixture: &LlamaFixture<'_>, ) -> Result<()> { - let context = context_after_decoding_text(fixture, LlamaPoolingType::Rank)?; + let context = context_after_decoding_text(fixture, LlamaPoolingType::Rank); assert_eq!( context.embeddings_seq_ith(0)?.len(), diff --git a/llama-cpp-bindings-tests/tests/embeddings_toggling.rs b/llama-cpp-bindings-tests/tests/embeddings_toggling.rs new file mode 100644 index 000000000..ab9d512b0 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/embeddings_toggling.rs @@ -0,0 +1,146 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const STATE: &str = "Shoes arrived two weeks late and in the wrong size."; +const ROW: &str = " Which team should handle this ticket?"; +const OTHER_STATE: &str = "The weather in Oslo is cold, dark and windy all week."; + +fn tokenize(fixture: &LlamaFixture<'_>, text: &str) -> Result> { + Ok(fixture + .model + .str_to_token(text, AddBos::Never, ParseSpecialTokens::Never)?) +} + +fn add_tokens( + batch: &mut LlamaBatch, + tokens: &[LlamaToken], + first_position: i32, + sequence_id: i32, + output_on_last_token: bool, +) -> Result<()> { + let last_index = tokens.len() - 1; + + for (index, token) in tokens.iter().enumerate() { + batch.add( + &SampledToken::Content(*token), + first_position + i32::try_from(index)?, + &[sequence_id], + output_on_last_token && index == last_index, + )?; + } + + Ok(()) +} + +fn decode_tokens( + context: &mut LlamaContext<'_>, + tokens: &[LlamaToken], + first_position: i32, + output_on_last_token: bool, +) -> Result { + let mut batch = LlamaBatch::new(ContextLimits::of(context)); + + add_tokens(&mut batch, tokens, first_position, 0, output_on_last_token)?; + context.decode(&batch)?; + + Ok(batch.n_tokens() - 1) +} + +fn final_hidden_state_context<'fixture>( + fixture: &'fixture LlamaFixture<'_>, + embeddings: bool, +) -> Result> { + Ok(LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_embeddings(embeddings) + .with_pooling_type(LlamaPoolingType::None), + )?) +} + +fn row_hidden_state_after_enabling_embeddings( + fixture: &LlamaFixture<'_>, + state: &[LlamaToken], + row: &[LlamaToken], +) -> Result> { + let mut context = final_hidden_state_context(fixture, false)?; + + decode_tokens(&mut context, state, 0, false)?; + context.set_embeddings(true); + + let row_last_index = decode_tokens(&mut context, row, i32::try_from(state.len())?, true)?; + + Ok(context.embeddings_ith(row_last_index)?.to_vec()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn final_hidden_states_after_enabling_embeddings_for_a_row_equal_those_of_an_embeddings_context( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let state = tokenize(fixture, STATE)?; + let row = tokenize(fixture, ROW)?; + let toggled_row_hidden_state = + row_hidden_state_after_enabling_embeddings(fixture, &state, &row)?; + + let mut embeddings_context = final_hidden_state_context(fixture, true)?; + + decode_tokens(&mut embeddings_context, &state, 0, false)?; + + let row_last_index = decode_tokens( + &mut embeddings_context, + &row, + i32::try_from(state.len())?, + true, + )?; + + assert_eq!( + toggled_row_hidden_state.as_slice(), + embeddings_context.embeddings_ith(row_last_index)?, + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn final_hidden_states_of_a_row_follow_the_state_decoded_before_it( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let row = tokenize(fixture, ROW)?; + + assert_ne!( + row_hidden_state_after_enabling_embeddings(fixture, &tokenize(fixture, STATE)?, &row)?, + row_hidden_state_after_enabling_embeddings( + fixture, + &tokenize(fixture, OTHER_STATE)?, + &row + )?, + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs new file mode 100644 index 000000000..25ee00af5 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/encode_batch_rejection.rs @@ -0,0 +1,75 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::batch_transfer_error::BatchTransferError; +use llama_cpp_bindings::error::encode_error::EncodeError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn encoding_more_tokens_than_the_micro_batch_is_refused(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture.model.str_to_token( + &"hello ".repeat(100), + AddBos::Never, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let n_tokens = batch.n_tokens(); + + assert_eq!( + context.encode(&batch), + Err(EncodeError::BatchExceedsMicroBatch { + n_tokens, + n_ubatch: context.n_ubatch(), + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn encoding_reports_an_exception_thrown_while_transferring_the_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_BATCH_EXT_ASSIGN_ENTRIES_THROWS, + ); + + assert_eq!( + context.encode(&batch), + Err(EncodeError::BatchTransfer(BatchTransferError::Reported { + message: injected_fault_message(), + })) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/encoder_requirement.rs b/llama-cpp-bindings-tests/tests/encoder_requirement.rs new file mode 100644 index 000000000..699fe54db --- /dev/null +++ b/llama-cpp-bindings-tests/tests/encoder_requirement.rs @@ -0,0 +1,67 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::encode_error::EncodeError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn encoding_with_a_decoder_only_model_reports_it_has_no_encoder( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let result = context.encode(&batch); + + assert_eq!(result, Err(EncodeError::ModelHasNoEncoder)); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn an_encoder_model_encodes_a_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + + let result = context.encode(&batch); + + assert_eq!(result, Ok(())); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs new file mode 100644 index 000000000..c621362f2 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/end_of_generation_token_detection.rs @@ -0,0 +1,55 @@ +use anyhow::Context; +use anyhow::Result; + +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn the_eos_token_ends_generation_in_every_section(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let eos = model.token_eos().context("model declares an EOS token")?; + + assert_eq!( + [ + SampledToken::Content(eos), + SampledToken::Reasoning(eos), + SampledToken::ToolCall(eos), + SampledToken::Undeterminable(eos), + ] + .map(|sampled_token| model.is_eog_token(&sampled_token)), + [true, true, true, true] + ); + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/generation_control.rs b/llama-cpp-bindings-tests/tests/generation_control.rs deleted file mode 100644 index 6c472f9f7..000000000 --- a/llama-cpp-bindings-tests/tests/generation_control.rs +++ /dev/null @@ -1,2400 +0,0 @@ -use std::ffi::CStr; -use std::io::Write; -use std::time::Duration; - -use anyhow::Context as _; -use anyhow::Result; -use anyhow::anyhow; -use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::GbnfValidationError; -use llama_cpp_bindings::GenerationProgress; -use llama_cpp_bindings::GrammarError; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::SamplerAcceptError; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::ggml_time_us; -use llama_cpp_bindings::json_schema_to_grammar; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::llguidance_sampler::create_llg_sampler; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::sampled_token_section::SampledTokenSection; -use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let tokens = model.str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - batch.add_sequence(&tokens, 0, false)?; - - context.decode(&mut batch)?; - - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; - - let result = sampler.sample(&context, batch.n_tokens() - 1); - - assert!(result.is_ok()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = "<|im_start|>user\nIs the sky blue? Answer yes or no.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - batch.add_sequence(&tokens, 0, false)?; - - context.decode(&mut batch)?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::grammar(model, r"root ::= [Yy] [Ee] [Ss] | [Nn] [Oo]", "root")?, - LlamaSampler::temp(0.8)?, - LlamaSampler::greedy()?, - ])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut outcomes = Vec::new(); - let turn = classifier.sample(&mut sampler, &context, batch.n_tokens() - 1, &mut outcomes)?; - classifier.finish(&mut outcomes); - - assert_eq!( - outcomes.len(), - 1, - "expected one finalised outcome after finishing" - ); - let outcome = &outcomes[0]; - - let raw_as_sampled = SampledToken::Content(turn.token); - assert!( - !model.is_eog_token(&raw_as_sampled), - "Grammar sampler should not allow EOS as first token" - ); - - let piece = outcome.piece.raw(); - let first_char = piece - .chars() - .next() - .ok_or_else(|| anyhow!("piece should have at least one character"))? - .to_lowercase() - .next() - .ok_or_else(|| anyhow!("lowercase iterator should yield a character"))?; - - assert!( - first_char == 'y' || first_char == 'n', - "Grammar should constrain first token to start with y/n, got: '{piece}'" - ); - assert_eq!( - classifier.usage().completion_tokens(), - 1, - "exactly one completion token sampled" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = "<|im_start|>user\nWhat is 2+2? Respond with a JSON object.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - batch.add_sequence(&tokens, 0, false)?; - - context.decode(&mut batch)?; - - let grammar_str = json_schema_to_grammar( - r#"{"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}"#, - )?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::grammar(model, &grammar_str, "root")?, - LlamaSampler::temp(0.8)?, - LlamaSampler::greedy()?, - ])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut outcomes = Vec::new(); - let turn = classifier.sample(&mut sampler, &context, batch.n_tokens() - 1, &mut outcomes)?; - classifier.finish(&mut outcomes); - - assert_eq!( - outcomes.len(), - 1, - "expected one finalised outcome after finishing" - ); - let outcome = &outcomes[0]; - - let raw_as_sampled = SampledToken::Content(turn.token); - assert!( - !model.is_eog_token(&raw_as_sampled), - "Grammar sampler should not allow EOS as first token" - ); - - let piece = outcome.piece.raw(); - - assert!( - piece.starts_with('{'), - "JSON schema grammar should constrain first token to start with '{{', got: '{piece}'" - ); - assert_eq!( - classifier.usage().completion_tokens(), - 1, - "exactly one completion token sampled" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn sample_with_grammar_produces_constrained_output_in_loop( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = "<|im_start|>user\nIs the sky blue? yes or no<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - context.decode(&mut batch)?; - classifier.commit_prompt_tokens(); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::grammar(model, r#"root ::= "yes" | "no""#, "root")?, - LlamaSampler::temp(0.8)?, - LlamaSampler::greedy()?, - ])?; - - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: 10, - } - .run()?; - - let lowercase = outcome.generated_raw.to_lowercase(); - assert!( - lowercase == "yes" || lowercase == "no", - "Grammar loop should produce 'yes' or 'no', got: '{}'", - outcome.generated_raw - ); - assert!( - outcome.eog_seen, - "loop must terminate via EOG once grammar accepts, not by exhausting the budget; outcome={outcome:?}" - ); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(outcome.observed_tool_call, 0); - assert!(outcome.observed_content > 0); - - let usage = classifier.into_usage(); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = - "<|im_start|>user\nSay hello<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - batch.add_sequence(&tokens, 0, false)?; - - context.decode(&mut batch)?; - - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut outcomes = Vec::new(); - let mut sampled_count: u64 = 0; - - for (position, _) in (batch.n_tokens()..).zip(0..5) { - let turn = classifier.sample(&mut sampler, &context, -1, &mut outcomes)?; - - if turn.progress == GenerationProgress::Ended { - break; - } - - sampled_count += 1; - - let raw_as_sampled = SampledToken::Content(turn.token); - - batch.clear(); - batch.add(&raw_as_sampled, position, &[0], true)?; - - context.decode(&mut batch)?; - } - - classifier.finish(&mut outcomes); - - assert!( - sampled_count > 0, - "Should produce at least one token without grammar" - ); - let usage = classifier.into_usage(); - assert!( - usage.completion_tokens() >= sampled_count, - "completion_tokens ({}) must include the {sampled_count} non-EOG samples", - usage.completion_tokens() - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn dry_sampler_with_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let breakers: Vec<&[u8]> = vec![b"\n", b"\t"]; - let _sampler = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn dry_sampler_with_null_byte_in_seq_breakers_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let breakers: Vec<&[u8]> = vec![b"hello\0world"]; - let result = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, breakers); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_returns_sampler_for_valid_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = LlamaSampler::grammar(fixture.model, "root ::= \"hello\"", "root"); - - assert!(sampler.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_returns_sampler_for_valid_grammar_with_patterns( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let patterns = vec!["\\{.*".to_owned()]; - let sampler = - LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); - - assert!(sampler.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_with_root_not_found_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let patterns = vec!["function".to_owned()]; - let result = - LlamaSampler::grammar_lazy(fixture.model, "expr ::= \"hello\"", "root", &patterns, &[]); - - assert_eq!(result.err(), Some(GrammarError::RootNotFound)); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_with_null_byte_in_pattern_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let patterns = vec!["hel\0lo".to_owned()]; - let result = - LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); - - assert_eq!( - result.err(), - Some(GrammarError::TriggerPatternContainsNul( - std::ffi::CString::new("hel\0lo").expect_err("the pattern carries a nul byte") - )) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_returns_sampler_for_valid_grammar_with_trigger_tokens( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let trigger_tokens = - fixture - .model - .str_to_token("{", AddBos::Never, ParseSpecialTokens::Always)?; - - assert!( - !trigger_tokens.is_empty(), - "the tokenizer must produce at least one trigger token" - ); - - let sampler = LlamaSampler::grammar_lazy( - fixture.model, - "root ::= \"hello\"", - "root", - &[], - &trigger_tokens, - ); - - assert!( - sampler.is_ok(), - "trigger tokens alone must be enough to build a lazy grammar sampler" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_with_null_byte_in_grammar_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = - LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hel\0lo\"", "root", &[], &[]); - - assert_eq!( - result.err(), - Some(GrammarError::GrammarContainsNul( - std::ffi::CString::new("root ::= \"hel\0lo\"") - .expect_err("the grammar carries a nul byte") - )) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn grammar_lazy_with_malformed_regex_returns_invalid_trigger_pattern( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let patterns = vec!["[".to_owned()]; - let result = - LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); - - assert!(matches!( - result, - Err(GrammarError::InvalidTriggerPattern { .. }), - )); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn llguidance_method_creates_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = LlamaSampler::llguidance(fixture.model, "regex", r"yes|no"); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn logit_bias_with_empty_biases_succeeds(_fixture: &LlamaFixture<'_>) -> Result<()> { - let result = LlamaSampler::logit_bias(0, &[]); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn dry_sampler_with_root_not_found_grammar_does_not_apply( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let breakers: Vec<&[u8]> = vec![b"\n"]; - let _sampler = LlamaSampler::dry(fixture.model, 1.5, 2.0, 128, 2, &breakers); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn accept_many_iterates_over_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; - let tokens = vec![ - fixture - .model - .token_bos() - .context("model declares a BOS token")?, - fixture - .model - .token_eos() - .context("model declares an EOS token")?, - ]; - - sampler.accept_many(&tokens)?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn with_tokens_returns_self_after_accepting_each_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; - let tokens = [ - fixture - .model - .token_bos() - .context("model declares a BOS token")?, - fixture - .model - .token_eos() - .context("model declares an EOS token")?, - ]; - - let _consumed = sampler.with_tokens(tokens.iter().copied())?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn accept_consumes_a_single_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; - - sampler.accept( - fixture - .model - .token_bos() - .context("model declares a BOS token")?, - )?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn accept_succeeds_for_a_valid_token(_fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; - - sampler.accept(LlamaToken::new(0))?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn apply_runs_sampler_over_token_data_array(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("Hi", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let mut data_array = context.token_data_array_ith(batch.n_tokens() - 1)?; - let sampler = LlamaSampler::greedy()?; - sampler.apply(&mut data_array)?; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, -)] -fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let tokens = fixture - .model - .str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; - let result = sampler.sample(&context, batch.n_tokens() - 1); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn raw_prompt_completion_with_timing(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - let mut ctx = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - ) - .with_context(|| "unable to create context")?; - - let prompt = "Hello my name is"; - let max_generated_tokens: i32 = 64; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let tokens_list = model - .str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always) - .with_context(|| format!("failed to tokenize {prompt}"))?; - let prompt_token_count = u64::try_from(tokens_list.len())?; - - let mut decoder = encoding_rs::UTF_8.new_decoder(); - - for token in &tokens_list { - eprint!( - "{}", - model.token_to_piece(&SampledToken::Content(*token), &mut decoder, true, None)? - ); - } - std::io::stderr().flush()?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens_list, 0, false)?; - - assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - ctx.decode(&mut batch) - .with_context(|| "llama_decode() failed")?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, prompt_token_count); - - let mut sampler = - LlamaSampler::chain_simple([LlamaSampler::dist(1234)?, LlamaSampler::greedy()?])?; - let initial_position = batch.n_tokens(); - let t_main_start = ggml_time_us(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut ctx, - batch: &mut batch, - initial_position, - max_generated_tokens, - } - .run()?; - let t_main_end = ggml_time_us(); - let duration = Duration::from_micros(u64::try_from(t_main_end - t_main_start)?); - let total_observed = - outcome.observed_content + outcome.observed_reasoning + outcome.observed_undeterminable; - - let tokens_per_second = f64::from(u32::try_from(total_observed)?) / duration.as_secs_f64(); - - eprintln!( - "\ndecoded {total_observed} tokens in {:.2} s, speed {tokens_per_second:.2} t/s", - duration.as_secs_f64(), - ); - - assert!( - !outcome.generated_raw.is_empty(), - "model should generate at least one token" - ); - assert_eq!( - outcome.observed_tool_call, 0, - "raw prompt without tool-call markers must not produce ToolCall tokens; \ - outcome={outcome:?}" - ); - assert!( - total_observed > 0, - "model must produce at least one classified token; outcome={outcome:?}" - ); - - let usage = classifier.into_usage(); - assert_eq!( - usage.prompt_tokens, prompt_token_count, - "prompt_tokens must equal the tokenizer's prompt length" - ); - assert_eq!( - usage.content_tokens, outcome.observed_content, - "content_tokens must equal observed Content variants" - ); - assert_eq!( - usage.reasoning_tokens, outcome.observed_reasoning, - "reasoning_tokens must equal observed Reasoning variants" - ); - assert_eq!( - usage.undeterminable_tokens, outcome.observed_undeterminable, - "undeterminable_tokens must equal observed Undeterminable variants" - ); - assert_eq!( - usage.tool_call_tokens, outcome.observed_tool_call, - "tool_call_tokens must equal observed ToolCall variants" - ); - assert_eq!( - usage.completion_tokens(), - total_observed, - "completion_tokens must equal Content + Reasoning + Undeterminable" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn chat_inference_produces_coherent_output(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let chat_template = model.chat_template(None)?; - let messages = vec![LlamaChatMessage::new( - "user".to_string(), - "Hello! How are you?".to_string(), - )?]; - let prompt = model.apply_chat_template(&chat_template, &messages, true, true)?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let tokens = model.str_to_token(&prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let prompt_token_count = u64::try_from(tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - assert_eq!(classifier.pending_prompt_tokens(), prompt_token_count); - assert_eq!(classifier.usage().prompt_tokens, 0); - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: 512, - } - .run()?; - - println!(); - - assert!( - !outcome.generated_raw.is_empty(), - "model should generate at least one token" - ); - let total_observed = - outcome.observed_content + outcome.observed_reasoning + outcome.observed_undeterminable; - assert!( - total_observed > 0, - "model must produce at least one classified token; outcome={outcome:?}" - ); - assert_eq!( - outcome.observed_tool_call, 0, - "chat without tool definitions must not produce ToolCall tokens; outcome={outcome:?}" - ); - - let usage = classifier.into_usage(); - - assert_eq!( - usage.prompt_tokens, prompt_token_count, - "prompt_tokens must equal the tokenizer's prompt length" - ); - assert_eq!( - usage.content_tokens, outcome.observed_content, - "content_tokens must equal observed Content variants" - ); - assert_eq!( - usage.reasoning_tokens, outcome.observed_reasoning, - "reasoning_tokens must equal observed Reasoning variants" - ); - assert_eq!( - usage.undeterminable_tokens, outcome.observed_undeterminable, - "undeterminable_tokens must equal observed Undeterminable variants" - ); - assert_eq!( - usage.completion_tokens(), - total_observed, - "completion_tokens must equal Content + Reasoning + Undeterminable" - ); - assert_eq!( - usage.tool_call_tokens, outcome.observed_tool_call, - "tool_call_tokens must equal observed ToolCall variants" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn json_schema_constrains_output(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - - let prompt = "The weather in Paris is sunny and 22 degrees. Extract as JSON:\n"; - - let mut ctx = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let tokens_list = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - let last_index = i32::try_from(tokens_list.len())? - 1; - - for (index, token) in (0_i32..).zip(&tokens_list) { - batch.add( - &SampledToken::Content(*token), - index, - &[0], - index == last_index, - )?; - } - - ctx.decode(&mut batch)?; - - let schema = r#"{ - "type": "object", - "properties": { - "city": { "type": "string" }, - "temperature": { "type": "number" } - }, - "required": ["city", "temperature"] -}"#; - - let llg_sampler = LlamaSampler::llguidance(model, "json", schema)?; - let mut sampler = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; - - let mut n_cur = batch.n_tokens(); - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let mut generated = String::new(); - - while n_cur <= 128 { - let token = SampledToken::Content(sampler.sample(&ctx, batch.n_tokens() - 1)?); - - if model.is_eog_token(&token) { - break; - } - - let output_string = model.token_to_piece(&token, &mut decoder, true, None)?; - generated.push_str(&output_string); - print!("{output_string}"); - std::io::stdout().flush()?; - - batch.clear(); - batch.add(&token, n_cur, &[0], true)?; - n_cur += 1; - ctx.decode(&mut batch)?; - } - - println!(); - - let parsed = serde_json::Deserializer::from_str(&generated) - .into_iter::() - .next() - .ok_or_else(|| anyhow!("model produced no JSON value"))??; - - assert!(parsed.get("city").is_some()); - assert!(parsed.get("temperature").is_some()); - - Ok(()) -} - -const JSON_SCHEMA: &str = - r#"{"type":"object","properties":{"answer":{"type":"string"}},"required":["answer"]}"#; -const REGEX_GRAMMAR: &str = r"yes|no"; -const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn creates_sampler_with_valid_json_schema(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "json", JSON_SCHEMA)?; - - assert!(!sampler.as_ptr().is_null()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn creates_sampler_with_valid_regex_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - assert!(!sampler.as_ptr().is_null()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn creates_sampler_with_valid_lark_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "lark", LARK_GRAMMAR)?; - - assert!(!sampler.as_ptr().is_null()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn returns_error_for_unknown_grammar_kind(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = create_llg_sampler(fixture.model, "not_a_real_kind", "anything"); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn returns_error_for_malformed_json_schema(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = create_llg_sampler(fixture.model, "json", "{this is not valid json"); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn returns_error_for_malformed_regex(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = create_llg_sampler(fixture.model, "regex", "[invalid"); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - let name_ptr = unsafe { llama_cpp_bindings_sys::llama_sampler_name(sampler.as_ptr()) }; - assert!(!name_ptr.is_null()); - let name = unsafe { CStr::from_ptr(name_ptr) }.to_str()?; - - assert_eq!(name, "llguidance"); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn clone_via_ffi_creates_independent_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { - let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - let cloned = unsafe { llama_cpp_bindings_sys::llama_sampler_clone(sampler.as_ptr()) }; - - assert!(!cloned.is_null()); - - unsafe { llama_cpp_bindings_sys::llama_sampler_free(cloned) }; - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let prompt = "Answer yes or no:"; - let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; - let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; - - let token = chain.sample(&context, batch.n_tokens() - 1)?; - assert!( - token.0 >= 0, - "grammar-constrained sampling must yield a valid token id without the grammar rejecting it" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn reset_rolls_back_accepted_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - let yes_tokens = - fixture - .model - .str_to_token("yes", AddBos::Never, ParseSpecialTokens::Always)?; - let first_allowed_token = *yes_tokens - .first() - .ok_or_else(|| anyhow!("the tokenizer must produce a token for \"yes\""))?; - - sampler - .accept(first_allowed_token) - .context("the grammar must accept the first token of an allowed word")?; - - sampler - .reset() - .context("reset must roll a healthy grammar matcher back to its initial state")?; - - assert_eq!( - sampler.accept(first_allowed_token), - Ok(()), - "after reset the same token must be acceptable again, which is what rollback means" - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - let huge_token = LlamaToken(i32::MAX - 1); - let result = sampler.accept(huge_token); - - let Err(SamplerAcceptError::GrammarCallbackFailed { message }) = result else { - panic!("expected a grammar callback failure, got {result:?}"); - }; - assert!(message.contains("token id 2147483646 out of range")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn approximate_tok_env_drives_consistent_grammar_constraint( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let first = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - let second = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; - - assert!(!first.as_ptr().is_null()); - assert!(!second.as_ptr().is_null()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 128, -)] -fn llguidance_chain_samples_a_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let tokens = model.str_to_token("Answer:", AddBos::Always, ParseSpecialTokens::Always)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - batch.add_sequence(&tokens, 0, false)?; - context.decode(&mut batch)?; - - let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; - let mut chain = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()?])?; - let token = chain.sample(&context, batch.n_tokens() - 1)?; - - assert!(token.0 >= 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn classifier_starts_in_pending_section_for_default_fixture( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let classifier = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Detect)?; - - assert_eq!(classifier.current_section(), SampledTokenSection::Pending); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn classifier_construction_is_idempotent_across_calls(fixture: &LlamaFixture<'_>) -> Result<()> { - let first = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Detect)?; - let second = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Detect)?; - - assert_eq!(first.current_section(), second.current_section()); - assert_eq!(first.usage(), second.usage()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn finishing_releases_an_unmatched_token_with_its_visible_and_raw_piece( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let [ordinary_token] = - model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] - else { - bail!("\"hello\" must be a single token"); - }; - - let mut outcomes = Vec::new(); - classifier.ingest(ordinary_token, &mut outcomes)?; - classifier.finish(&mut outcomes); - - assert_eq!(outcomes.len(), 1); - let outcome = &outcomes[0]; - assert!(matches!( - outcome.sampled_token, - SampledToken::Undeterminable(_) - )); - assert_eq!(outcome.piece.visible(), outcome.piece.raw()); - assert_eq!(classifier.usage().undeterminable_tokens, 1); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn ingest_counts_every_token_through_the_end_of_generation( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let [ordinary_token] = - model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] - else { - bail!("\"hello\" must be a single token"); - }; - let mut outcomes = Vec::new(); - - classifier.ingest(ordinary_token, &mut outcomes)?; - - assert_eq!( - classifier.ingest( - model.token_eos().context("model declares an EOS token")?, - &mut outcomes - )?, - GenerationProgress::Ended - ); - - assert_eq!(classifier.usage().undeterminable_tokens, 2); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn ingest_unmatched_prompt_tokens_does_not_record_usage(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let usage_before = *classifier.usage(); - - classifier.ingest_prompt_token(model.token_bos().context("model declares a BOS token")?); - classifier.ingest_prompt_tokens(&[ - model.token_eos().context("model declares an EOS token")?, - model.token_nl().context("model declares a newline token")?, - ]); - - assert_eq!(*classifier.usage(), usage_before); - assert_eq!(classifier.current_section(), SampledTokenSection::Pending); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(8)?, 1); - - classifier.feed_prompt_to_batch( - &mut batch, - model.token_bos().context("model declares a BOS token")?, - 0, - &[0], - false, - )?; - classifier.feed_prompt_to_batch( - &mut batch, - model.token_eos().context("model declares an EOS token")?, - 1, - &[0], - false, - )?; - - assert_eq!(classifier.pending_prompt_tokens(), 2); - assert_eq!(batch.n_tokens(), 2); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(8)?, 1); - - let tokens = vec![ - model.token_bos().context("model declares a BOS token")?, - model.token_eos().context("model declares an EOS token")?, - model.token_nl().context("model declares a newline token")?, - ]; - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - assert_eq!(classifier.pending_prompt_tokens(), 3); - assert_eq!(batch.n_tokens(), 3); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(8)?, 1); - - classifier.feed_prompt_to_batch( - &mut batch, - model.token_bos().context("model declares a BOS token")?, - 0, - &[0], - false, - )?; - classifier.feed_prompt_to_batch( - &mut batch, - model.token_eos().context("model declares an EOS token")?, - 1, - &[0], - false, - )?; - - let promoted = classifier.commit_prompt_tokens(); - - assert_eq!(promoted, 2); - assert_eq!(classifier.pending_prompt_tokens(), 0); - assert_eq!(classifier.usage().prompt_tokens, 2); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn discard_pending_prompt_tokens_clears_count_without_recording_usage( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(8)?, 1); - - classifier.feed_prompt_to_batch( - &mut batch, - model.token_bos().context("model declares a BOS token")?, - 0, - &[0], - false, - )?; - - let discarded = classifier.discard_pending_prompt_tokens(); - - assert_eq!(discarded, 1); - assert_eq!(classifier.pending_prompt_tokens(), 0); - assert_eq!(classifier.usage().prompt_tokens, 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn diagnose_tool_call_synthetic_renders_applies_the_template_to_both_probes( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let renders = fixture.model.diagnose_tool_call_synthetic_renders()?; - - assert!( - !renders.without_tools.is_empty(), - "the probe render without tool calls must not be empty" - ); - assert!( - !renders.with_tools.is_empty(), - "the probe render with a tool call must not be empty" - ); - for render in [&renders.without_tools, &renders.with_tools] { - assert!( - render.contains("Please use the tool"), - "each probe render must contain the synthetic user turn, which is what proves the \ - template was applied; got: {render:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_grammar_with_a_token_reference_constrains_the_first_token( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let [think_token] = - model.str_to_token("", AddBos::Never, ParseSpecialTokens::Always)?[..] - else { - bail!(" must be a single token"); - }; - let mut context = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let prompt_tokens = model.str_to_token( - "<|im_start|>user\nSay hi<|im_end|>\n<|im_start|>assistant\n", - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(128)?, 1); - - batch.add_sequence(&prompt_tokens, 0, false)?; - context.decode(&mut batch)?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::grammar(model, r#"root ::= "x""#, "root")?, - LlamaSampler::greedy()?, - ])?; - - assert_eq!(sampler.sample(&context, batch.n_tokens() - 1)?, think_token); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_grammar_without_its_root_reports_the_missing_root( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - assert_eq!( - LlamaSampler::grammar(fixture.model, r#"answer ::= "x""#, "root").err(), - Some(GrammarError::RootNotFound) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_grammar_with_a_syntax_error_reports_it(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_eq!( - LlamaSampler::grammar(fixture.model, "root ::= (", "root").err(), - Some(GrammarError::GrammarRejected( - GbnfValidationError::SyntaxError - )) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/generation_end_classification.rs b/llama-cpp-bindings-tests/tests/generation_end_classification.rs new file mode 100644 index 000000000..e6db6af12 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/generation_end_classification.rs @@ -0,0 +1,140 @@ +use anyhow::Result; + +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampled_token_section::SampledTokenSection; +use llama_cpp_bindings_tests::ingested_generation::ingested_generation; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const END_OF_GENERATION_TOKENS: u64 = 1; + +fn visible_text(outcomes: &[IngestOutcome]) -> String { + outcomes + .iter() + .map(|outcome| outcome.piece.visible()) + .collect() +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_classifier_ends_generation_without_emitting_the_end_of_generation_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const GENERATED_TEXT: &str = "The answer is four."; + + let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + + assert_eq!( + visible_text(&ingested_generation( + &mut classifier, + fixture.model, + GENERATED_TEXT + )), + GENERATED_TEXT + ); + + Ok(()) +} + +fn visible_text_in(outcomes: &[IngestOutcome], section: SampledTokenSection) -> String { + outcomes + .iter() + .filter(|outcome| outcome.sampled_token.section() == section) + .map(|outcome| outcome.piece.visible()) + .collect() +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn an_answer_stopped_inside_unclosed_reasoning_releases_its_unsettled_tail_as_content( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const SETTLED_REASONING: &str = "Let me think"; + const UNSETTLED_TAIL: &str = ", +) -> Result<()> { + const GENERATED_TEXT: &str = r#"{"name": "get_weather", "arguments": {"location": "Paris"}}"#; + + let session = user_turn_chat_session(fixture.model, "What is 2 + 2?", "[]", false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let outcomes = ingested_generation(&mut classifier, fixture.model, GENERATED_TEXT); + + assert_eq!( + [ + visible_text_in(&outcomes, SampledTokenSection::Reasoning), + visible_text_in(&outcomes, SampledTokenSection::Content), + visible_text_in(&outcomes, SampledTokenSection::ToolCall), + visible_text_in(&outcomes, SampledTokenSection::Pending), + ], + [ + String::new(), + GENERATED_TEXT.to_owned(), + String::new(), + String::new() + ] + ); + assert_eq!( + classifier.usage().content_tokens, + u64::try_from(outcomes.len())? + END_OF_GENERATION_TOKENS + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/grammar_sampler.rs b/llama-cpp-bindings-tests/tests/grammar_sampler.rs new file mode 100644 index 000000000..9dc7b7191 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/grammar_sampler.rs @@ -0,0 +1,322 @@ +use std::ffi::CString; + +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::grammar_error::GrammarError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_bindings_tests::logits_left_by::logits_left_by; +use llama_cpp_bindings_tests::single_token::single_token; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const YES_TOKEN_AND_END_OF_GENERATION: u64 = 2; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn grammar_sampler_constrains_the_first_token_of_the_answer_to_yes( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let prompt = "<|im_start|>user\nIs the sky blue? Answer yes or no.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&tokens, 0, false)?; + + context.decode(&batch)?; + + let mut sampler = LlamaSampler::chain([ + LlamaSampler::grammar(model, r"root ::= [Yy] [Ee] [Ss] | [Nn] [Oo]", "root")?, + LlamaSampler::temp(0.8)?, + LlamaSampler::greedy()?, + ])?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut outcomes = Vec::new(); + let turn = classifier.sample(&mut sampler, &context, batch.n_tokens() - 1, &mut outcomes)?; + classifier.finish(&mut outcomes)?; + + assert_eq!( + outcomes.len(), + 1, + "expected one finalised outcome after finishing" + ); + let outcome = &outcomes[0]; + + let raw_as_sampled = SampledToken::Content(turn.token); + assert!( + !model.is_eog_token(&raw_as_sampled), + "Grammar sampler should not allow EOS as first token" + ); + + let piece = outcome.piece.raw(); + + assert_eq!(piece, "Yes"); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn grammar_constrained_chat_answer_is_yes_content_that_ends_generation( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let session = closed_reasoning_chat_session(model, "Is the sky blue? yes or no", "[]"); + let tokens = session.tokenize_prompt()?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; + + context.decode(&batch)?; + classifier.commit_prompt_tokens(); + + let mut sampler = LlamaSampler::chain([ + LlamaSampler::grammar(model, r#"root ::= "yes" | "no""#, "root")?, + LlamaSampler::temp(0.8)?, + LlamaSampler::greedy()?, + ])?; + + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position, + max_generated_tokens: 10, + } + .run(); + + assert_eq!(outcome.generated.text(), "yes"); + assert!( + outcome.eog_seen, + "loop must terminate via EOG once grammar accepts, not by exhausting the budget; outcome={outcome:?}" + ); + assert_eq!(outcome.observed_reasoning, 0); + assert_eq!(outcome.observed_undeterminable, 0); + assert_eq!(outcome.observed_tool_call, 0); + assert_eq!(outcome.observed_content, YES_TOKEN_AND_END_OF_GENERATION); + + let usage = classifier.into_usage(); + assert_eq!(usage.completion_tokens(), outcome.observed_content); + assert_eq!(usage.reasoning_tokens, 0); + assert_eq!(usage.undeterminable_tokens, 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn a_grammar_sampler_masks_tokens_that_cannot_start_the_grammar( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let sampler = LlamaSampler::grammar(model, "root ::= \"hello\"", "root")?; + + assert_eq!( + logits_left_by( + &sampler, + &[single_token(model, "hello"), single_token(model, "yes")] + ), + [0.0, f32::NEG_INFINITY] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_grammar_with_a_token_reference_constrains_the_first_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let think_token = single_token(model, ""); + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let prompt_tokens = model.str_to_token( + "<|im_start|>user\nSay hi<|im_end|>\n<|im_start|>assistant\n", + AddBos::Never, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&prompt_tokens, 0, false)?; + context.decode(&batch)?; + + let mut sampler = LlamaSampler::chain([ + LlamaSampler::grammar(model, r#"root ::= "x""#, "root")?, + LlamaSampler::greedy()?, + ])?; + + assert_eq!(sampler.sample(&context, batch.n_tokens() - 1)?, think_token); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_grammar_without_its_root_reports_the_missing_root(fixture: &LlamaFixture<'_>) { + assert_eq!( + LlamaSampler::grammar(fixture.model, r#"answer ::= "x""#, "root").err(), + Some(GrammarError::RootNotFound) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_grammar_with_a_syntax_error_reports_it(fixture: &LlamaFixture<'_>) { + assert_eq!( + LlamaSampler::grammar(fixture.model, "root ::= (", "root").err(), + Some(GrammarError::GrammarRejected( + GbnfValidationError::SyntaxError + )) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_grammar_with_a_nul_byte_is_refused(fixture: &LlamaFixture<'_>) { + assert_eq!( + LlamaSampler::grammar(fixture.model, "root ::= \"a\0\"", "root").err(), + CString::new("root ::= \"a\0\"") + .err() + .map(GrammarError::GrammarContainsNul) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_valid_grammar_llama_cpp_refuses_to_load_is_reported_malformed(fixture: &LlamaFixture<'_>) { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GRAMMAR_RETURNS_NULL, + ); + + assert_eq!( + LlamaSampler::grammar(fixture.model, "root ::= \"a\"", "root").err(), + Some(GrammarError::GrammarMalformed) + ); +} diff --git a/llama-cpp-bindings-tests/tests/hidden_states.rs b/llama-cpp-bindings-tests/tests/hidden_states.rs deleted file mode 100644 index 2c66b1875..000000000 --- a/llama-cpp-bindings-tests/tests/hidden_states.rs +++ /dev/null @@ -1,440 +0,0 @@ -use anyhow::Result; -use llama_cpp_bindings::EmbeddingsError; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::context::params::LlamaPoolingType; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -const STATE: &str = "Shoes arrived two weeks late and in the wrong size."; -const FIRST_BRANCH: &str = " Which team should handle this ticket?"; -const SECOND_BRANCH: &str = " Is the customer angry about the delay?"; -const OTHER_STATE: &str = "The weather in Oslo is cold, dark and windy all week."; - -fn tokenize(fixture: &LlamaFixture<'_>, text: &str) -> Result> { - Ok(fixture - .model - .str_to_token(text, AddBos::Never, ParseSpecialTokens::Never)?) -} - -fn hidden_state_context<'fixture>( - fixture: &'fixture LlamaFixture<'_>, -) -> Result> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::None) - .with_kv_unified(true), - )?; - - context.enable_masked_nextn_embeddings()?; - - Ok(context) -} - -fn add_tokens( - batch: &mut LlamaBatch<'_>, - tokens: &[LlamaToken], - first_position: i32, - sequence_id: i32, - output_on_last_token: bool, -) -> Result<()> { - let last_index = tokens.len() - 1; - - for (index, token) in tokens.iter().enumerate() { - batch.add( - &SampledToken::Content(*token), - first_position + i32::try_from(index)?, - &[sequence_id], - output_on_last_token && index == last_index, - )?; - } - - Ok(()) -} - -fn hidden_state_after( - fixture: &LlamaFixture<'_>, - state: &[LlamaToken], - branch: &[LlamaToken], -) -> Result> { - let mut context = hidden_state_context(fixture)?; - let mut state_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - add_tokens(&mut state_batch, state, 0, 0, false)?; - context.decode(&mut state_batch)?; - - let mut branch_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - add_tokens( - &mut branch_batch, - branch, - i32::try_from(state.len())?, - 0, - true, - )?; - context.decode(&mut branch_batch)?; - - Ok(context - .nextn_embeddings_ith(branch_batch.n_tokens() - 1)? - .to_vec()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn masked_nextn_embeddings_equal_the_final_hidden_states(fixture: &LlamaFixture<'_>) -> Result<()> { - let tokens = tokenize(fixture, STATE)?; - let last_index = i32::try_from(tokens.len() - 1)?; - - let mut embedding_context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_embeddings(true) - .with_pooling_type(LlamaPoolingType::None), - )?; - let mut embedding_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - embedding_batch.add_sequence(&tokens, 0, true)?; - embedding_context.decode(&mut embedding_batch)?; - - let mut nextn_context = hidden_state_context(fixture)?; - let mut nextn_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - nextn_batch.add_sequence(&tokens, 0, true)?; - nextn_context.decode(&mut nextn_batch)?; - - assert_eq!( - nextn_context.nextn_embeddings_ith(last_index)?, - embedding_context.embeddings_ith(last_index)?, - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn masked_nextn_embeddings_exist_only_for_output_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let tokens = tokenize(fixture, STATE)?; - let mut context = hidden_state_context(fixture)?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - add_tokens(&mut batch, &tokens, 0, 0, true)?; - context.decode(&mut batch)?; - - assert_eq!( - context.nextn_embeddings_ith(0), - Err(EmbeddingsError::NextnEmbeddingUnavailable { token_index: 0 }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn nextn_embeddings_refuse_a_pooling_context(fixture: &LlamaFixture<'_>) -> Result<()> { - let mut context = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_pooling_type(LlamaPoolingType::Mean), - )?; - - assert_eq!( - context.enable_masked_nextn_embeddings(), - Err(EmbeddingsError::NextnEmbeddingsRequireNonePooling { - pooling_type: LlamaPoolingType::Mean, - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn nextn_embeddings_are_unavailable_until_enabled(fixture: &LlamaFixture<'_>) -> Result<()> { - let context = fixture.build_context()?; - - assert_eq!( - context.nextn_embeddings_ith(0), - Err(EmbeddingsError::NextnEmbeddingsNotEnabled) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, - n_seq_max = 2, -)] -fn a_released_fork_leaves_its_shared_state_as_an_independent_decode_would( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let state = tokenize(fixture, STATE)?; - let first_branch = tokenize(fixture, FIRST_BRANCH)?; - let second_branch = tokenize(fixture, SECOND_BRANCH)?; - let branch_position = i32::try_from(state.len())?; - - let mut context = hidden_state_context(fixture)?; - let mut state_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - add_tokens(&mut state_batch, &state, 0, 0, false)?; - context.decode(&mut state_batch)?; - context.copy_kv_cache_seq(0, 1, None, None)?; - - let mut first_branch_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - add_tokens( - &mut first_branch_batch, - &first_branch, - branch_position, - 1, - true, - )?; - context.decode(&mut first_branch_batch)?; - - let first_branch_hidden_state = context - .nextn_embeddings_ith(first_branch_batch.n_tokens() - 1)? - .to_vec(); - - context.clear_kv_cache_seq(Some(1), None, None)?; - - let mut second_branch_batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - add_tokens( - &mut second_branch_batch, - &second_branch, - branch_position, - 0, - true, - )?; - context.decode(&mut second_branch_batch)?; - - assert_eq!( - first_branch_hidden_state, - hidden_state_after(fixture, &state, &first_branch)?, - ); - assert_eq!( - context.nextn_embeddings_ith(second_branch_batch.n_tokens() - 1)?, - hidden_state_after(fixture, &state, &second_branch)?.as_slice(), - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn tokenizing_without_special_token_parsing_keeps_control_markup_as_text( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let markup = "<|fim_prefix|>"; - let parsed = fixture - .model - .str_to_token(markup, AddBos::Never, ParseSpecialTokens::Always)?; - let plain = fixture - .model - .str_to_token(markup, AddBos::Never, ParseSpecialTokens::Never)?; - - assert_eq!(parsed.len(), 1); - assert!(plain.len() > 1); - assert!(!plain.contains(&parsed[0])); - - Ok(()) -} - -fn decode_tokens( - context: &mut LlamaContext<'_>, - tokens: &[LlamaToken], - first_position: i32, - output_on_last_token: bool, -) -> Result { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - - add_tokens(&mut batch, tokens, first_position, 0, output_on_last_token)?; - context.decode(&mut batch)?; - - Ok(batch.n_tokens() - 1) -} - -fn final_hidden_state_context<'fixture>( - fixture: &'fixture LlamaFixture<'_>, - embeddings: bool, -) -> Result> { - Ok(LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params) - .into_llama_context_params() - .with_embeddings(embeddings) - .with_pooling_type(LlamaPoolingType::None), - )?) -} - -fn row_hidden_state_after_enabling_embeddings( - fixture: &LlamaFixture<'_>, - state: &[LlamaToken], - row: &[LlamaToken], -) -> Result> { - let mut context = final_hidden_state_context(fixture, false)?; - - decode_tokens(&mut context, state, 0, false)?; - context.set_embeddings(true); - - let row_last_index = decode_tokens(&mut context, row, i32::try_from(state.len())?, true)?; - - Ok(context.embeddings_ith(row_last_index)?.to_vec()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn final_hidden_states_after_enabling_embeddings_for_a_row_equal_those_of_an_embeddings_context( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let state = tokenize(fixture, STATE)?; - let row = tokenize(fixture, FIRST_BRANCH)?; - let toggled_row_hidden_state = - row_hidden_state_after_enabling_embeddings(fixture, &state, &row)?; - - let mut embeddings_context = final_hidden_state_context(fixture, true)?; - - decode_tokens(&mut embeddings_context, &state, 0, false)?; - - let row_last_index = decode_tokens( - &mut embeddings_context, - &row, - i32::try_from(state.len())?, - true, - )?; - - assert_eq!( - toggled_row_hidden_state.as_slice(), - embeddings_context.embeddings_ith(row_last_index)?, - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn disabling_embeddings_makes_token_embeddings_unavailable( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let state = tokenize(fixture, STATE)?; - let mut context = final_hidden_state_context(fixture, true)?; - let last_index = decode_tokens(&mut context, &state, 0, true)?; - - context.set_embeddings(false); - - assert_eq!( - context.embeddings_ith(last_index), - Err(EmbeddingsError::NotEnabled) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn final_hidden_states_of_a_row_follow_the_state_decoded_before_it( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let row = tokenize(fixture, FIRST_BRANCH)?; - - assert_ne!( - row_hidden_state_after_enabling_embeddings(fixture, &tokenize(fixture, STATE)?, &row)?, - row_hidden_state_after_enabling_embeddings( - fixture, - &tokenize(fixture, OTHER_STATE)?, - &row - )?, - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 512, -)] -fn nextn_embeddings_are_refused_and_stay_disabled_on_a_model_without_nextn_hidden_states( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mut context = final_hidden_state_context(fixture, false)?; - - assert_eq!( - context.enable_masked_nextn_embeddings(), - Err(EmbeddingsError::NextnEmbeddingsUnsupported) - ); - assert_eq!( - context.nextn_embeddings_ith(0), - Err(EmbeddingsError::NextnEmbeddingsNotEnabled) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/image_question_answering.rs b/llama-cpp-bindings-tests/tests/image_question_answering.rs new file mode 100644 index 000000000..7cba5800d --- /dev/null +++ b/llama-cpp-bindings-tests/tests/image_question_answering.rs @@ -0,0 +1,132 @@ +use anyhow::Context; +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn a_question_about_an_image_is_answered_with_its_image_tokens_counted_as_input( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let mut ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + ) + .with_context(|| "unable to create llama context")?; + + assert!( + mtmd_ctx + .capabilities(model) + .contains(MtmdCapability::Vision), + "model should support vision input" + ); + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .with_context(|| "image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str) + .with_context(|| "failed to load image from file")?; + + let session = media_question_chat_session(model, "What animals do you see in this image?")?; + let input_text = MtmdInputText { + text: session.prompt()?, + add_special: true, + parse_special: true, + }; + + let chunks = mtmd_ctx + .tokenize(input_text, &[&bitmap]) + .with_context(|| "failed to tokenize multimodal input")?; + + assert!( + !chunks.is_empty(), + "tokenization should produce at least one chunk" + ); + + let expected = ChunkTokenBreakdown::from_chunks(&chunks); + + assert!( + expected.image > 0, + "vision input must produce at least one image chunk" + ); + + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let n_past = classifier + .eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut ctx, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + ) + .with_context(|| "failed to evaluate chunks")?; + + { + let usage = classifier.usage(); + assert_eq!(usage.prompt_tokens, expected.text); + assert_eq!(usage.input_image_tokens, expected.image); + assert_eq!(usage.input_audio_tokens, expected.audio); + } + + let mut sampler = LlamaSampler::greedy()?; + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + let totals = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut ctx, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: 512, + } + .run(); + + assert!( + !totals.generated.text().is_empty(), + "model should generate at least one token from image input" + ); + + let usage = classifier.into_usage(); + assert_eq!(usage.prompt_tokens, expected.text); + assert_eq!(usage.input_image_tokens, expected.image); + assert_eq!(usage.input_audio_tokens, expected.audio); + assert_eq!(usage.content_tokens, totals.observed_content); + assert_eq!(usage.reasoning_tokens, totals.observed_reasoning); + assert_eq!( + usage.completion_tokens(), + totals.observed_content + totals.observed_reasoning + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs new file mode 100644 index 000000000..0895dceda --- /dev/null +++ b/llama-cpp-bindings-tests/tests/json_schema_constrained_generation.rs @@ -0,0 +1,232 @@ +use anyhow::Result; +use serde_json::Value; +use serde_json::json; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::json_schema_to_grammar::json_schema_to_grammar; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const MAX_ANSWER_TOKENS: i32 = 64; + +fn json_schema_constrained_answer(fixture: &LlamaFixture<'_>) -> Result { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let session = + closed_reasoning_chat_session(model, "What is 2+2? Respond with a JSON object.", "[]"); + let tokens = session.tokenize_prompt()?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let mut classifier = SampledTokenClassifier::undeterminable(model); + + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; + context.decode(&batch)?; + classifier.commit_prompt_tokens(); + + let grammar_str = json_schema_to_grammar( + r#"{"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}"#, + )?; + + let mut sampler = LlamaSampler::chain([ + LlamaSampler::grammar(model, &grammar_str, "root")?, + LlamaSampler::temp(0.8)?, + LlamaSampler::greedy()?, + ])?; + + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run(); + + assert!( + outcome.eog_seen, + "the answer must end once the schema is satisfied; outcome={outcome:?}" + ); + + Ok(outcome.generated.text().parse::()?) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn deepseek_r1_8b_answers_the_json_schema_with_four(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + json_schema_constrained_answer(fixture)?, + json!({"answer": "4"}) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn glm47_flash_answers_the_json_schema_with_a_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + json_schema_constrained_answer(fixture)?, + json!({"answer": "The sum of 2 and 2 is 4."}) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen35_answers_the_json_schema_with_four(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + json_schema_constrained_answer(fixture)?, + json!({"answer": "4"}) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen36_answers_the_json_schema_with_an_empty_string(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!( + json_schema_constrained_answer(fixture)?, + json!({"answer": ""}) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn an_llguidance_json_schema_constrains_the_answer_to_the_schema( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let backend = fixture.backend; + + let prompt = "The weather in Paris is sunny and 22 degrees. Extract as JSON:\n"; + + let mut ctx = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let tokens_list = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&ctx)); + let last_index = i32::try_from(tokens_list.len())? - 1; + + for (index, token) in (0_i32..).zip(&tokens_list) { + batch.add( + &SampledToken::Content(*token), + index, + &[0], + index == last_index, + )?; + } + + ctx.decode(&batch)?; + + let schema = r#"{ + "type": "object", + "properties": { + "city": { "type": "string" }, + "temperature": { "type": "number" } + }, + "required": ["city", "temperature"], + "additionalProperties": false +}"#; + + let llg_sampler = LlamaSampler::llguidance(model, "json", schema)?; + let mut sampler = LlamaSampler::chain([llg_sampler, LlamaSampler::greedy()?])?; + + let initial_position = batch.n_tokens(); + let outcome = ClassifySampleLoop { + classifier: &mut SampledTokenClassifier::undeterminable(model), + sampler: &mut sampler, + context: &mut ctx, + batch: &mut batch, + initial_position, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run(); + + assert!( + outcome.eog_seen, + "the answer must end once the schema is satisfied; outcome={outcome:?}" + ); + assert_eq!( + outcome.generated.text().parse::()?, + json!({"city": "Paris", "temperature": 22}) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs new file mode 100644 index 000000000..ee6b020cd --- /dev/null +++ b/llama-cpp-bindings-tests/tests/lazy_grammar_sampler.rs @@ -0,0 +1,143 @@ +use std::ffi::CString; + +use anyhow::Result; + +use llama_cpp_bindings::error::grammar_error::GrammarError; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::logits_left_by::logits_left_by; +use llama_cpp_bindings_tests::single_token::single_token; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn a_lazy_grammar_awaiting_its_trigger_pattern_leaves_candidates_unconstrained( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let patterns = vec!["\\{.*".to_owned()]; + let sampler = LlamaSampler::grammar_lazy(model, "root ::= \"hello\"", "root", &patterns, &[])?; + + assert_eq!( + logits_left_by( + &sampler, + &[single_token(model, "hello"), single_token(model, "yes")] + ), + [0.0, 0.0] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_without_its_root_reports_the_missing_root(fixture: &LlamaFixture<'_>) { + let patterns = vec!["function".to_owned()]; + let result = + LlamaSampler::grammar_lazy(fixture.model, "expr ::= \"hello\"", "root", &patterns, &[]); + + assert_eq!(result.err(), Some(GrammarError::RootNotFound)); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_refuses_a_trigger_pattern_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + let patterns = vec!["hel\0lo".to_owned()]; + let result = + LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); + + assert_eq!( + result.err(), + CString::new("hel\0lo") + .err() + .map(GrammarError::TriggerPatternContainsNul) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn a_lazy_grammar_awaiting_its_trigger_token_leaves_candidates_unconstrained( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let sampler = LlamaSampler::grammar_lazy( + model, + "root ::= \"hello\"", + "root", + &[], + &[single_token(model, "{")], + )?; + + assert_eq!( + logits_left_by( + &sampler, + &[single_token(model, "hello"), single_token(model, "yes")] + ), + [0.0, 0.0] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_refuses_a_grammar_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + let result = + LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hel\0lo\"", "root", &[], &[]); + + assert_eq!( + result.err(), + CString::new("root ::= \"hel\0lo\"") + .err() + .map(GrammarError::GrammarContainsNul) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn grammar_lazy_with_malformed_regex_returns_invalid_trigger_pattern(fixture: &LlamaFixture<'_>) { + let patterns = vec!["[".to_owned()]; + let result = + LlamaSampler::grammar_lazy(fixture.model, "root ::= \"hello\"", "root", &patterns, &[]); + + assert!(matches!( + result, + Err(GrammarError::InvalidTriggerPattern { .. }), + )); +} diff --git a/llama-cpp-bindings-tests/tests/llguidance_sampler.rs b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs new file mode 100644 index 000000000..d97373ff9 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/llguidance_sampler.rs @@ -0,0 +1,709 @@ +use std::ffi::CStr; + +use anyhow::Context as _; +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::grammar_error::GrammarError; +use llama_cpp_bindings::error::grammar_runtime_error::GrammarRuntimeError; +use llama_cpp_bindings::error::sample_error::SampleError; +use llama_cpp_bindings::error::sampler_accept_error::SamplerAcceptError; +use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::llguidance_sampler::create_llg_sampler; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::params::LlamaModelParams; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_tests::logits_left_by::logits_left_by; +use llama_cpp_bindings_tests::single_token::single_token; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const JSON_SCHEMA: &str = + r#"{"type":"object","properties":{"answer":{"type":"string"}},"required":["answer"]}"#; +const REGEX_GRAMMAR: &str = r"yes|no"; + +const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; + +fn out_of_vocabulary_failure() -> String { + GrammarRuntimeError::InternalParserError { + message: "token id 2147483646 out of range".to_owned(), + } + .to_string() +} + +fn yes_no_and_hello(model: &LlamaModel) -> [LlamaToken; 3] { + [ + single_token(model, "yes"), + single_token(model, "no"), + single_token(model, "hello"), + ] +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_json_schema_sampler_masks_tokens_that_cannot_start_the_schema( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let sampler = create_llg_sampler(model, "json", JSON_SCHEMA)?; + + assert_eq!( + logits_left_by( + &sampler, + &[single_token(model, "{"), single_token(model, "hello")] + ), + [0.0, f32::NEG_INFINITY] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_regex_sampler_masks_tokens_that_cannot_start_a_match( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let sampler = LlamaSampler::llguidance(model, "regex", REGEX_GRAMMAR)?; + + assert_eq!( + logits_left_by(&sampler, &yes_no_and_hello(model)), + [0.0, 0.0, f32::NEG_INFINITY] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_lark_sampler_masks_tokens_that_cannot_start_the_grammar( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let sampler = create_llg_sampler(model, "lark", LARK_GRAMMAR)?; + + assert_eq!( + logits_left_by(&sampler, &yes_no_and_hello(model)), + [0.0, 0.0, f32::NEG_INFINITY] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn an_unknown_grammar_kind_is_reported_as_an_invalid_grammar(fixture: &LlamaFixture<'_>) { + assert_eq!( + create_llg_sampler(fixture.model, "not_a_real_kind", "anything").err(), + Some(GrammarError::LlguidanceGrammarInvalid { + message: "unknown constraint type: not_a_real_kind".to_owned(), + }) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_json_schema_that_is_not_json_is_reported_as_an_invalid_grammar(fixture: &LlamaFixture<'_>) { + assert_eq!( + create_llg_sampler(fixture.model, "json", "{this is not valid json").err(), + Some(GrammarError::LlguidanceGrammarInvalid { + message: "key must be a string at line 1 column 2".to_owned(), + }) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_malformed_regex_leaves_llguidance_without_a_parser(fixture: &LlamaFixture<'_>) -> Result<()> { + let Err(GrammarError::LlguidanceParserUnavailable { .. }) = + create_llg_sampler(fixture.model, "regex", "[invalid") + else { + bail!("a regex with an unclosed character class must leave llguidance without a parser"); + }; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { + let sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + + let name_ptr = unsafe { llama_cpp_bindings_sys::llama_sampler_name(sampler.as_ptr()) }; + assert!(!name_ptr.is_null()); + let name = unsafe { CStr::from_ptr(name_ptr) }.to_str()?; + + assert_eq!(name, "llguidance"); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_sampler_cloned_through_ffi_keeps_its_own_grammar_state( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; + let mut cloned = unsafe { + LlamaSampler::from_raw( + llama_cpp_bindings_sys::llama_sampler_clone(sampler.as_ptr()), + "llguidance", + ) + }?; + + assert_eq!(cloned.accept(single_token(model, "yes")), Ok(())); + assert_eq!(sampler.accept(single_token(model, "no")), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn llguidance_chain_samples_yes_as_the_first_answer_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let backend = fixture.backend; + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let prompt = "Answer yes or no:"; + let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let llg_sampler = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; + let mut chain = LlamaSampler::chain([llg_sampler, LlamaSampler::greedy()?])?; + + let token = chain.sample(&context, batch.n_tokens() - 1)?; + let piece = model.token_to_piece( + &SampledToken::Content(token), + &mut encoding_rs::UTF_8.new_decoder(), + false, + None, + )?; + + assert_eq!(piece, "yes"); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn reset_rolls_back_accepted_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + + let yes_tokens = + fixture + .model + .str_to_token("yes", AddBos::Never, ParseSpecialTokens::Always)?; + let first_allowed_token = *yes_tokens + .first() + .context("the tokenizer must produce a token for \"yes\"")?; + + sampler + .accept(first_allowed_token) + .context("the grammar must accept the first token of an allowed word")?; + + sampler + .reset() + .context("reset must roll a healthy grammar matcher back to its initial state")?; + + assert_eq!( + sampler.accept(first_allowed_token), + Ok(()), + "after reset the same token must be acceptable again, which is what rollback means" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn accept_maps_an_out_of_vocabulary_token_to_grammar_callback_failure( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut sampler = create_llg_sampler(fixture.model, "regex", REGEX_GRAMMAR)?; + + assert_eq!( + sampler.accept(LlamaToken(i32::MAX - 1)), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn samplers_sharing_the_cached_tokenizer_environment_mask_alike( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let candidates = yes_no_and_hello(model); + let first = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; + let second = create_llg_sampler(model, "regex", REGEX_GRAMMAR)?; + + assert_eq!( + logits_left_by(&first, &candidates), + logits_left_by(&second, &candidates) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn sampling_through_a_grammar_in_an_error_state_reports_the_grammar_failure( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut context = fixture.build_context()?; + let tokens = model.str_to_token( + "Answer yes or no:", + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let mut chain = LlamaSampler::chain([ + create_llg_sampler(model, "regex", REGEX_GRAMMAR)?, + LlamaSampler::greedy()?, + ])?; + + assert_eq!( + chain.accept(LlamaToken(i32::MAX - 1)), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + assert_eq!( + chain.sample(&context, batch.n_tokens() - 1), + Err(SampleError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + + Ok(()) +} + +fn freshly_loaded(fixture: &LlamaFixture<'_>) -> Result { + Ok(LlamaModel::load_from_file( + fixture.backend, + fixture.model_path, + &LlamaModelParams::default(), + )?) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn creating_a_sampler_reports_a_vocabulary_llama_cpp_cannot_render( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = freshly_loaded(fixture)?; + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_TOKEN_TO_PIECE_FAILS); + + assert_eq!( + create_llg_sampler(&model, "regex", REGEX_GRAMMAR).err(), + Some(GrammarError::TokEnvUnavailable( + TokenToStringError::InsufficientBufferSpace(-34) + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn the_tokenizer_environment_reports_a_control_token_llama_cpp_cannot_render( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = freshly_loaded(fixture)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SPECIAL_TOKEN_TO_PIECE_FAILS); + + assert_eq!( + model.approximate_tok_env().err(), + Some(TokenToStringError::InsufficientBufferSpace(-34)) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs new file mode 100644 index 000000000..2903f7641 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/logit_bias_sampler.rs @@ -0,0 +1,26 @@ +use anyhow::Result; + +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_tests::logits_left_by::logits_left_by; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn a_logit_bias_sampler_with_an_empty_bias_list_leaves_candidates_unchanged( + _fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + logits_left_by(&LlamaSampler::logit_bias(0, &[])?, &[LlamaToken::new(0)]), + [0.0] + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/logits_availability.rs b/llama-cpp-bindings-tests/tests/logits_availability.rs new file mode 100644 index 000000000..28e3de348 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/logits_availability.rs @@ -0,0 +1,116 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::llama_pooling_type::LlamaPoolingType; +use llama_cpp_bindings::error::logits_error::LogitsError; +use llama_cpp_bindings::error::sample_error::SampleError; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; +use llama_cpp_bindings_tests::context_after_decoding_text::context_after_decoding_text; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn sampling_after_a_pooled_embeddings_decode_is_refused_instead_of_aborting( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); + let mut sampler = LlamaSampler::greedy()?; + + assert_eq!( + sampler.sample(&context, -1), + Err(SampleError::Logits( + LogitsError::PooledEmbeddingsProduceNoLogits { + pooling_type: LlamaPoolingType::Mean, + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("LiquidAI/LFM2-ColBERT-350M-GGUF", "LFM2-ColBERT-350M-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn a_model_whose_embeddings_graph_has_no_logits_head_produces_no_logits( + fixture: &LlamaFixture<'_>, +) { + let context = context_after_decoding_text(fixture, LlamaPoolingType::None); + + assert_eq!( + context.get_logits_ith(-1), + Err(LogitsError::GraphProducesNoLogits) + ); +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn every_logits_accessor_refuses_a_pooled_embeddings_decode(fixture: &LlamaFixture<'_>) { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); + + assert_eq!( + [ + context.get_logits_ith(-1).err(), + context.candidates_ith(-1).err(), + context.token_data_array_ith(-1).err(), + context + .fill_token_data_array_ith(-1, &mut LlamaTokenDataArray::new(Vec::new(), false)) + .err(), + ], + [(); 4].map(|()| { + Some(LogitsError::PooledEmbeddingsProduceNoLogits { + pooling_type: LlamaPoolingType::Mean, + }) + }) + ); +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn classified_sampling_after_a_pooled_embeddings_decode_is_refused( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = context_after_decoding_text(fixture, LlamaPoolingType::Mean); + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + assert_eq!( + classifier + .sample(&mut LlamaSampler::greedy()?, &context, -1, &mut Vec::new()) + .err(), + Some(SampleError::Logits( + LogitsError::PooledEmbeddingsProduceNoLogits { + pooling_type: LlamaPoolingType::Mean, + } + )) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/logits_retrieval.rs b/llama-cpp-bindings-tests/tests/logits_retrieval.rs new file mode 100644 index 000000000..f78d74ae0 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/logits_retrieval.rs @@ -0,0 +1,93 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::logits_error::LogitsError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn a_logits_row_holds_one_logit_per_vocabulary_token(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let last_index = i32::try_from(tokens.len() - 1)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let logits = context.get_logits_ith(last_index)?; + + assert_eq!(logits.len(), usize::try_from(fixture.model.n_vocab())?); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn get_logits_ith_returns_token_not_initialized_for_unknown_index( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let result = context.get_logits_ith(7); + + assert_eq!(result, Err(LogitsError::TokenNotInitialized(7))); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn reading_a_logits_row_reports_a_row_llama_cpp_does_not_return( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_GET_LOGITS_ITH_RETURNS_NULL); + + assert_eq!( + context.get_logits_ith(-1).err(), + Some(LogitsError::NullLogits) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/lora_adapters.rs b/llama-cpp-bindings-tests/tests/lora_adapters.rs new file mode 100644 index 000000000..cebe2f2f9 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/lora_adapters.rs @@ -0,0 +1,182 @@ +use std::env; +use std::ffi::CString; +use std::ffi::OsStr; +use std::fs; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; +use std::path::PathBuf; + +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::llama_lora_adapter_init_error::LlamaLoraAdapterInitError; +use llama_cpp_bindings::error::llama_lora_adapters_error::LlamaLoraAdaptersError; +use llama_cpp_bindings::model::lora_adapter_scale::LoraAdapterScale; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; +use llama_cpp_test_model_provisioning::download_model::download_model; + +#[llama_test( + model_source = HuggingFace( + "Qwen/Qwen2.5-7B-Instruct-GGUF", + "qwen2.5-7b-instruct-q2_k.gguf" + ), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn set_lora_adapters_applies_a_real_adapter(fixture: &LlamaFixture<'_>) -> Result<()> { + let adapter_path = download_model( + "ggml-org/LoRA-Deepthink-Reasoning-Qwen2.5-7B-Instruct-Q8_0-GGUF", + "Deepthink-Reasoning-Adapter-q8_0.gguf", + )?; + let adapter = fixture.model.lora_adapter_init(&adapter_path)?; + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!( + context.set_lora_adapters(&[LoraAdapterScale { + adapter: &adapter, + scale: 1.0, + }]), + Ok(()) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn set_lora_adapters_accepts_an_empty_adapter_set(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + assert_eq!(context.set_lora_adapters(&[]), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace( + "Qwen/Qwen2.5-7B-Instruct-GGUF", + "qwen2.5-7b-instruct-q2_k.gguf" + ), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn setting_lora_adapters_reports_a_status_llama_cpp_returns( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let adapter_path = download_model( + "ggml-org/LoRA-Deepthink-Reasoning-Qwen2.5-7B-Instruct-Q8_0-GGUF", + "Deepthink-Reasoning-Adapter-q8_0.gguf", + )?; + let adapter = fixture.model.lora_adapter_init(&adapter_path)?; + let context = fixture.build_context()?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SET_ADAPTERS_LORA_FAILS); + + assert_eq!( + context.set_lora_adapters(&[LoraAdapterScale { + adapter: &adapter, + scale: 1.0, + }]), + Err(LlamaLoraAdaptersError::ErrorResult(-1)) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn lora_adapter_init_with_a_missing_file_reports_it_not_found(fixture: &LlamaFixture<'_>) { + let result = fixture + .model + .lora_adapter_init("/nonexistent/path/lora.gguf"); + assert_eq!( + result.unwrap_err(), + LlamaLoraAdapterInitError::FileNotFound(PathBuf::from("/nonexistent/path/lora.gguf")) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn lora_adapter_init_with_an_invalid_gguf_reports_it_unloadable( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let dummy_path = env::temp_dir().join("llama_test_dummy_lora.gguf"); + fs::write(&dummy_path, b"not a valid gguf")?; + + let result = fixture.model.lora_adapter_init(&dummy_path); + + assert_eq!(result.unwrap_err(), LlamaLoraAdapterInitError::Unloadable); + fs::remove_file(&dummy_path)?; + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn lora_adapter_init_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) { + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); + let result = fixture.model.lora_adapter_init(non_utf8_path); + + assert_eq!( + result.unwrap_err(), + LlamaLoraAdapterInitError::PathToStrError(non_utf8_path.to_path_buf()) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn lora_adapter_init_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + assert_eq!( + fixture + .model + .lora_adapter_init("/adapters/adapter\0.gguf") + .err(), + CString::new("/adapters/adapter\0.gguf") + .err() + .map(LlamaLoraAdapterInitError::NullError) + ); +} diff --git a/llama-cpp-bindings-tests/tests/main.rs b/llama-cpp-bindings-tests/tests/main.rs index ab13f7b94..8eb28ab1b 100644 --- a/llama-cpp-bindings-tests/tests/main.rs +++ b/llama-cpp-bindings-tests/tests/main.rs @@ -1,23 +1,84 @@ -#![expect( - clippy::unnecessary_wraps, - reason = "harness trial functions must return Result to satisfy LlamaTestFn registration" -)] -mod backend_initialization; -mod chat_protocol; -mod context_after_decoding_text; -mod context_state; +mod approximate_tok_env_eos_selection; +mod backend_device_listing; +mod batch_context_limits; +mod chat_answer_parsing; +mod chat_prompt_rendering; +mod chat_session_input_validation; +mod chat_session_llama_cpp_failures; +mod chat_template_lookup; +mod chat_template_override; +mod completion_usage_accounting; +mod context_creation_limits; +mod context_dimensions; +mod debug_formatting; +mod decode_abort_control; +mod decode_batch_rejection; +mod detokenization; +mod dry_sampler; mod embedding_availability; -mod embedding_models; +mod embedding_generation; +mod embedding_retrieval_errors; mod embedding_row_width; -mod generation_control; -mod hidden_states; +mod embeddings_toggling; +mod encode_batch_rejection; +mod encoder_requirement; +mod end_of_generation_token_detection; +mod generation_end_classification; +mod grammar_sampler; +mod image_question_answering; +mod json_schema_constrained_generation; +mod lazy_grammar_sampler; +mod llguidance_sampler; +mod logit_bias_sampler; +mod logits_availability; +mod logits_retrieval; +mod lora_adapters; +mod memory_availability; +mod memory_occupancy; +mod memory_position_bounds; +mod memory_position_shift_rope_compatibility; +mod memory_priming; +mod memory_sequence_edits; +mod memory_sequence_id_bounds; +mod model_architecture; mod model_derived_data_caching; -mod model_introspection; +mod model_hyperparameters; mod model_loading_errors; +mod model_metadata; mod multimodal_audio; +mod multimodal_bitmap_construction; +mod multimodal_chunk_copy; +mod multimodal_chunk_encoding; +mod multimodal_chunk_evaluation; +mod multimodal_chunk_tokenization; mod multimodal_fusion; +mod multimodal_projector_capabilities; +mod multimodal_projector_loading; mod multimodal_prompt_parts_tokenization; -mod multimodal_vision; -mod structured_chat_output; +mod multimodal_reasoning_classification; +mod multimodal_usage_accounting; +mod parameter_fitting; +mod prompt_token_accounting; +mod reasoning_classification; +mod reranking; +mod sampled_token_ingestion; +mod sampler_token_acceptance; +mod session_file_path_validation; +mod session_file_token_capacity; +mod session_files; +mod special_tokens; +mod state_snapshots; +mod token_attributes; +mod token_candidates; +mod tokenization; +mod tool_call_classification; +mod tool_call_grammar; +mod tool_call_parsing; +mod unconstrained_token_sampling; +mod vocabulary_bounds; +mod vocabulary_enumeration; +mod vocabulary_type; -llama_cpp_test_harness::llama_tests_main!(); +use llama_cpp_test_harness::llama_tests_main; + +llama_tests_main!(); diff --git a/llama-cpp-bindings-tests/tests/memory_availability.rs b/llama-cpp-bindings-tests/tests/memory_availability.rs new file mode 100644 index 000000000..77a06015d --- /dev/null +++ b/llama-cpp-bindings-tests/tests/memory_availability.rs @@ -0,0 +1,127 @@ +use std::num::NonZeroU8; + +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; +use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; +use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; +use llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError; +use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; +use llama_cpp_bindings_tests::two_sequence_context::two_sequence_context; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn an_encoder_context_keeps_no_memory(fixture: &LlamaFixture<'_>) -> Result<()> { + assert!(!fixture.build_context()?.has_memory()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, + embeddings = true, +)] +fn a_causal_embedding_context_keeps_memory(fixture: &LlamaFixture<'_>) -> Result<()> { + assert!(fixture.build_context()?.has_memory()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_threads_batch = 8, + embeddings = true, +)] +fn memory_edits_without_a_memory_module_trivially_succeed( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + context.memory_clear(); + + assert_eq!(context.memory_seq_keep(0), Ok(())); + assert_eq!(context.memory_seq_rm(Some(0), None, None), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_threads_batch = 8, + embeddings = true, +)] +fn memory_operations_needing_a_memory_module_report_its_absence( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = two_sequence_context(fixture)?; + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; + + assert_eq!( + context.memory_seq_cp(0, 1, None, None), + Err(MemorySeqCpError::MemoryHandleUnavailable) + ); + assert_eq!( + context.memory_seq_add(0, None, None, 1), + Err(MemorySeqAddError::MemoryHandleUnavailable) + ); + assert_eq!( + context.memory_seq_div(0, None, None, divisor), + Err(MemorySeqDivError::MemoryHandleUnavailable) + ); + assert_eq!( + context.memory_seq_pos_max(0), + Err(MemorySeqPosMaxError::MemoryHandleUnavailable) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("nomic-ai/nomic-embed-text-v1.5-GGUF", "nomic-embed-text-v1.5.Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_threads_batch = 8, + embeddings = true, +)] +fn removing_a_sequence_without_a_memory_module_still_validates_its_id( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_rm(Some(u32::MAX), None, None).err(), + i32::try_from(u32::MAX) + .err() + .map(MemorySeqRmError::SeqIdTooLarge) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/memory_occupancy.rs b/llama-cpp-bindings-tests/tests/memory_occupancy.rs new file mode 100644 index 000000000..469e14ed5 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/memory_occupancy.rs @@ -0,0 +1,153 @@ +use anyhow::Result; + +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings_tests::prime_memory::prime_memory; +use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; +use llama_cpp_bindings_tests::two_sequence_context::two_sequence_context; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const PRIMED_TEXT: &str = "Hello world"; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_clear_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + context.memory_clear(); + assert_eq!(context.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_pos_max_is_the_position_of_the_last_decoded_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let primed_tokens = + fixture + .model + .str_to_token(PRIMED_TEXT, AddBos::Always, ParseSpecialTokens::Always)?; + + prime_memory_with(fixture, &mut context, PRIMED_TEXT)?; + + assert_eq!( + context.memory_seq_pos_max(0)?, + i32::try_from(primed_tokens.len())? - 1 + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_keep_removes_every_other_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = two_sequence_context(fixture)?; + + prime_memory(fixture, &mut context)?; + + let primed_position_max = context.memory_seq_pos_max(0)?; + + context.memory_seq_cp(0, 1, None, None)?; + context.memory_seq_keep(1)?; + + assert_eq!(context.memory_seq_pos_max(0)?, -1); + assert_eq!(context.memory_seq_pos_max(1)?, primed_position_max); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/memory_position_bounds.rs b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs new file mode 100644 index 000000000..511b0a1f3 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/memory_position_bounds.rs @@ -0,0 +1,390 @@ +use std::num::NonZeroU8; + +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; +use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; +use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; +use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; +use llama_cpp_bindings_tests::two_sequence_context::two_sequence_context; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_cp_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = two_sequence_context(fixture)?; + + let result = context.memory_seq_cp(0, 1, Some(u32::MAX), None); + + assert_eq!( + result.unwrap_err(), + MemorySeqCpError::P0TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_cp_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = two_sequence_context(fixture)?; + + let result = context.memory_seq_cp(0, 1, Some(0), Some(u32::MAX)); + + assert_eq!( + result.unwrap_err(), + MemorySeqCpError::P1TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_rm_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.memory_seq_rm(Some(0), Some(u32::MAX), None); + + assert_eq!( + result.unwrap_err(), + MemorySeqRmError::P0TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_rm_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.memory_seq_rm(Some(0), Some(0), Some(u32::MAX)); + + assert_eq!( + result.unwrap_err(), + MemorySeqRmError::P1TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.memory_seq_add(0, Some(u32::MAX), None, 1); + + assert_eq!( + result.unwrap_err(), + MemorySeqAddError::P0TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.memory_seq_add(0, Some(0), Some(u32::MAX), 1); + + assert_eq!( + result.unwrap_err(), + MemorySeqAddError::P1TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; + let result = context.memory_seq_div(0, Some(u32::MAX), None, divisor); + + assert_eq!( + result.unwrap_err(), + MemorySeqDivError::P0TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; + let result = context.memory_seq_div(0, Some(0), Some(u32::MAX), divisor); + + assert_eq!( + result.unwrap_err(), + MemorySeqDivError::P1TooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs new file mode 100644 index 000000000..fc87dd200 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/memory_position_shift_rope_compatibility.rs @@ -0,0 +1,169 @@ +use std::num::NonZeroU8; + +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; +use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; +use llama_cpp_bindings::error::unknown_rope_type::UnknownRopeType; +use llama_cpp_bindings_tests::prime_memory::prime_memory; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_add_refuses_an_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + let result = context.memory_seq_add(0, Some(0), None, 1); + + assert_eq!(result.unwrap_err(), MemorySeqAddError::IncompatibleRopeType); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_div_refuses_an_mrope_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; + let result = context.memory_seq_div(0, Some(0), None, divisor); + + assert_eq!(result.unwrap_err(), MemorySeqDivError::IncompatibleRopeType); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_add_shifts_the_positions_of_a_standard_rope_model( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + let primed_position_max = context.memory_seq_pos_max(0)?; + + context.memory_seq_add(0, Some(0), None, 1)?; + + assert_eq!(context.memory_seq_pos_max(0)?, primed_position_max + 1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_div_divides_the_positions_of_a_standard_rope_model( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + let primed_position_max = context.memory_seq_pos_max(0)?; + let divisor = NonZeroU8::new(2).context("2 is non-zero")?; + + context.memory_seq_div(0, Some(0), None, divisor)?; + + assert_eq!( + context.memory_seq_pos_max(0)?, + primed_position_max / i32::from(divisor.get()) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_add_refuses_a_model_whose_rope_type_llama_cpp_does_not_define( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_ROPE_TYPE_UNKNOWN); + + assert_eq!( + context.memory_seq_add(0, None, None, 1), + Err(MemorySeqAddError::UnknownRopeType(UnknownRopeType( + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE + 1 + ))) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_div_refuses_a_model_whose_rope_type_llama_cpp_does_not_define( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + let _fault = ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_ROPE_TYPE_UNKNOWN); + + assert_eq!( + context.memory_seq_div(0, None, None, NonZeroU8::MIN), + Err(MemorySeqDivError::UnknownRopeType(UnknownRopeType( + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE + 1 + ))) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/memory_priming.rs b/llama-cpp-bindings-tests/tests/memory_priming.rs new file mode 100644 index 000000000..a8d84c4c7 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/memory_priming.rs @@ -0,0 +1,83 @@ +use std::ffi::CString; + +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::batch_add_error::BatchAddError; +use llama_cpp_bindings::error::decode_error::DecodeError; +use llama_cpp_bindings::error::string_to_token_error::StringToTokenError; +use llama_cpp_bindings_tests::prime_memory::prime_memory; +use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 64, +)] +fn priming_memory_surfaces_a_tokenization_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + let Err(failure) = prime_memory_with(fixture, &mut context, "Hello\0world") else { + bail!("an interior NUL byte must refuse the tokenization"); + }; + + assert_eq!( + Some(failure.downcast::()?), + CString::new("Hello\0world") + .err() + .map(StringToTokenError::NulError) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 64, +)] +fn priming_memory_surfaces_a_batch_add_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + prime_memory_with(fixture, &mut context, &"word ".repeat(300)) + .err() + .and_then(|error| error.downcast_ref::().cloned()), + Some(BatchAddError::BatchFull { n_batch: 256 }), + "more tokens than one decode takes must surface an add-sequence error" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 64, +)] +fn priming_memory_surfaces_a_decode_error(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + let Err(failure) = prime_memory(fixture, &mut context) else { + bail!("decoding the primed sequence again from its first position must fail"); + }; + + assert_eq!( + failure.downcast::()?, + DecodeError::BatchInvalid + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs new file mode 100644 index 000000000..552c557b8 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/memory_sequence_edits.rs @@ -0,0 +1,185 @@ +use anyhow::Result; + +use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; +use llama_cpp_bindings_tests::prime_memory::prime_memory; +use llama_cpp_bindings_tests::two_sequence_context::two_sequence_context; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_rm_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + assert_eq!(context.memory_seq_rm(Some(0), Some(0), Some(1)), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_cp_copies_a_whole_primed_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = two_sequence_context(fixture)?; + + prime_memory(fixture, &mut context)?; + + assert_eq!(context.memory_seq_cp(0, 1, None, None), Ok(())); + assert_eq!( + context.memory_seq_pos_max(1)?, + context.memory_seq_pos_max(0)? + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_cp_accepts_an_explicit_range(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = two_sequence_context(fixture)?; + + prime_memory(fixture, &mut context)?; + + assert_eq!(context.memory_seq_cp(0, 1, Some(0), Some(1)), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_recurrent_memory_refuses_to_remove_part_of_a_sequence( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + assert_eq!( + context.memory_seq_rm(Some(0), Some(1), Some(2)), + Err(MemorySeqRmError::PartialSequenceNotRemoved { + seq_id: 0, + p0: 1, + p1: 2, + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn removing_from_every_sequence_empties_the_primed_kv_cache( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + prime_memory(fixture, &mut context)?; + + assert_eq!(context.memory_seq_rm(None, None, None), Ok(())); + assert_eq!(context.memory_seq_pos_max(0)?, -1); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs new file mode 100644 index 000000000..9a2a10784 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/memory_sequence_id_bounds.rs @@ -0,0 +1,261 @@ +use std::num::NonZeroU8; + +use anyhow::Result; + +use llama_cpp_bindings::error::memory_seq_add_error::MemorySeqAddError; +use llama_cpp_bindings::error::memory_seq_cp_error::MemorySeqCpError; +use llama_cpp_bindings::error::memory_seq_div_error::MemorySeqDivError; +use llama_cpp_bindings::error::memory_seq_pos_max_error::MemorySeqPosMaxError; +use llama_cpp_bindings::error::memory_seq_rm_error::MemorySeqRmError; +use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_pos_max_rejects_out_of_range_sequence(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + let result = context.memory_seq_pos_max(999); + + assert_eq!( + result, + Err(MemorySeqPosMaxError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 999, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_rm_rejects_sequence_id_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let result = context.memory_seq_rm(Some(u32::MAX), None, None); + + assert_eq!( + result.unwrap_err(), + MemorySeqRmError::SeqIdTooLarge( + i32::try_from(u32::MAX).expect_err("u32::MAX does not fit into i32") + ) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_cp_rejects_a_destination_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_cp(0, 1, None, None), + Err(MemorySeqCpError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_keep_rejects_a_sequence_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_keep(1), + Err(SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_add_rejects_a_negative_sequence_id(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_add(-1, None, None, 1), + Err(MemorySeqAddError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: -1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_div_rejects_a_sequence_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_div(1, None, None, NonZeroU8::MIN), + Err(MemorySeqDivError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_rm_rejects_a_sequence_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_rm(Some(1), None, None), + Err(MemorySeqRmError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn memory_seq_cp_rejects_a_source_outside_the_context_sequences( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context.memory_seq_cp(1, 0, None, None), + Err(MemorySeqCpError::SequenceIdOutOfRange( + SequenceIdOutOfRange { + seq_id: 1, + n_seq_max: 1 + } + )) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/model_architecture.rs b/llama-cpp-bindings-tests/tests/model_architecture.rs new file mode 100644 index 000000000..a16739b82 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/model_architecture.rs @@ -0,0 +1,129 @@ +use llama_cpp_bindings::model::rope_type::RopeType; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn is_recurrent_returns_false_for_transformer(fixture: &LlamaFixture<'_>) { + assert!(!fixture.model.is_recurrent()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn is_hybrid_returns_false_for_non_hybrid_default_models(fixture: &LlamaFixture<'_>) { + assert!( + !fixture.model.is_hybrid(), + "DeepSeek-R1-Distill-Llama-8B and GLM-4.7-Flash are pure transformers, not hybrid; got is_hybrid=true" + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) { + assert!( + fixture.model.is_hybrid(), + "Qwen 3.5 and Qwen 3.6 default GGUFs are reported as hybrid by llama.cpp; got is_hybrid=false" + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn rope_type_reports_normal_rope_for_a_llama_architecture_model(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.rope_type(), Ok(RopeType::Norm)); +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn rope_type_reports_normal_rope_for_glm47_flash(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.rope_type(), Ok(RopeType::Norm)); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn rope_type_reports_interleaved_mrope_for_qwen_default_models(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.rope_type(), Ok(RopeType::IMRope)); +} diff --git a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs index 59bacd40a..7fdc0fccf 100644 --- a/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs +++ b/llama-cpp-bindings-tests/tests/model_derived_data_caching.rs @@ -1,14 +1,14 @@ use std::ptr; use std::sync::Arc; -use anyhow::Context; use anyhow::Result; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; + +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, @@ -25,38 +25,15 @@ fn approximate_tok_env_is_cached_across_calls(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128 -)] -fn streaming_markers_are_cached_across_calls(fixture: &LlamaFixture<'_>) -> Result<()> { - let first = fixture.model.streaming_markers()?; - let second = fixture.model.streaming_markers()?; - - assert!(Arc::ptr_eq(&first, &second)); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 2048, n_batch = 512, n_ubatch = 128 )] -fn reasoning_markers_are_cached_across_calls(fixture: &LlamaFixture<'_>) -> Result<()> { - let first = fixture - .model - .reasoning_markers()? - .context("Qwen3.5 must expose reasoning markers")?; - let second = fixture - .model - .reasoning_markers()? - .context("Qwen3.5 must expose reasoning markers")?; +fn chat_templates_are_prepared_once_per_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let first = fixture.model.chat_templates()?; + let second = fixture.model.chat_templates()?; assert!(ptr::eq(first, second)); diff --git a/llama-cpp-bindings-tests/tests/model_hyperparameters.rs b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs new file mode 100644 index 000000000..1535003f9 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/model_hyperparameters.rs @@ -0,0 +1,51 @@ +use anyhow::Result; + +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn model_reports_positive_core_hyperparameters(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + + assert!(model.n_vocab() > 0); + assert!(model.n_embd() > 0); + assert!(model.n_params() > 0); + assert!(model.n_ctx_train()? > 0); + assert!(model.n_layer()? > 0); + assert!(model.n_head()? > 0); + assert!(model.n_head_kv()? > 0); + assert!(model.size() > 0); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/model_introspection.rs b/llama-cpp-bindings-tests/tests/model_introspection.rs deleted file mode 100644 index 8340a3f6f..000000000 --- a/llama-cpp-bindings-tests/tests/model_introspection.rs +++ /dev/null @@ -1,1857 +0,0 @@ -use std::ffi::CString; -use std::num::NonZeroU16; -use std::num::NonZeroU32; -use std::pin::pin; - -use anyhow::Context; -use anyhow::Result; -use anyhow::anyhow; -use llama_cpp_bindings::SampledToken; -use llama_cpp_bindings::TokenAttrError; -use llama_cpp_bindings::TokenOutsideVocabulary; -use llama_cpp_bindings::TokenToStringError; -use llama_cpp_bindings::context::params::LlamaContextParams; -use llama_cpp_bindings::max_devices; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::model::params::LlamaModelParams; -use llama_cpp_bindings::model::params::fit_extra_model::FitExtraModel; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn model_loads_with_valid_metadata(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - - assert!(model.n_vocab() > 0); - assert!(model.n_embd() > 0); - assert!(model.n_params() > 0); - assert!(model.n_ctx_train()? > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_layer_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.n_layer()? > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_head_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.n_head()? > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_head_kv_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.n_head_kv()? > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn model_size_returns_nonzero(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.size() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_recurrent_returns_false_for_transformer(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(!fixture.model.is_recurrent()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_hybrid_returns_false_for_non_hybrid_default_models(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!( - !fixture.model.is_hybrid(), - "DeepSeek-R1-Distill-Llama-8B and GLM-4.7-Flash are pure transformers, not hybrid; got is_hybrid=true" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!( - fixture.model.is_hybrid(), - "Qwen 3.5 and Qwen 3.6 default GGUFs are reported as hybrid by llama.cpp; got is_hybrid=false" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn rope_type_returns_a_known_variant_for_rope_carrying_default_models( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - use llama_cpp_bindings::model::rope_type::RopeType; - let rope = fixture.model.rope_type(); - assert!( - matches!( - rope, - Some(RopeType::Norm | RopeType::NeoX | RopeType::MRope | RopeType::Vision) - ), - "rope_type must be a known variant for DeepSeek and GLM-4.7; got {rope:?}" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn rope_type_returns_none_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> Result<()> { - let rope = fixture.model.rope_type(); - assert!( - rope.is_none(), - "Qwen 3.5 and Qwen 3.6 default GGUFs do not expose a rope_type in their metadata; got {rope:?}" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn vocab_type_returns_a_known_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - use llama_cpp_bindings::model::vocab_type::VocabType; - let vocab = fixture.model.vocab_type()?; - assert!( - matches!(vocab, VocabType::BPE | VocabType::SPM), - "vocab_type must be a known variant; got {vocab:?}" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.meta_count() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_key_by_index_returns_valid_key(fixture: &LlamaFixture<'_>) -> Result<()> { - let key = fixture.model.meta_key_by_index(0)?; - assert_ne!(key, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_by_index_returns_valid_value(fixture: &LlamaFixture<'_>) -> Result<()> { - let value = fixture.model.meta_val_str_by_index(0)?; - assert_ne!(value, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_key_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture.model.meta_key_by_index(999_999); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture.model.meta_val_str_by_index(999_999); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let first_key = model.meta_key_by_index(0)?; - let value = model.meta_val_str(&first_key)?; - assert_ne!(value, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_with_long_value_triggers_buffer_resize(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let count = model.meta_count(); - - for index in 0..count { - let key = model.meta_key_by_index(index); - let value = model.meta_val_str_by_index(index); - assert!(key.is_ok()); - assert!(value.is_ok()); - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn meta_val_str_with_null_byte_in_key_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture.model.meta_val_str("key\0with_null"); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> { - let model_path_utf8 = fixture - .model_path - .to_str() - .ok_or_else(|| anyhow!("model path is not valid UTF-8"))?; - let model_path_c = CString::new(model_path_utf8)?; - - let mut params = pin!(LlamaModelParams::default()); - let mut context_params = LlamaContextParams::default(); - let mut margins = vec![0usize; max_devices()]; - - let result = params.as_mut().fit_params( - &model_path_c, - &mut context_params, - &mut margins, - 512, - None, - llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, - ); - - let fit = result.map_err(|fit_error| anyhow!("fit_params failed: {fit_error:?}"))?; - assert!(fit.n_ctx > 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn fit_params_aligns_the_extra_model_context_with_the_fitted_one( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model_path_utf8 = fixture - .model_path - .to_str() - .ok_or_else(|| anyhow!("model path is not valid UTF-8"))?; - let model_path_c = CString::new(model_path_utf8)?; - - let mut params = pin!(LlamaModelParams::default()); - let mut context_params = LlamaContextParams::default(); - let mut margins = vec![0usize; max_devices()]; - - let unfittable_extra_n_ctx = NonZeroU32::MIN; - let mut extra_params = pin!(LlamaModelParams::default()); - let mut extra_context_params = - LlamaContextParams::default().with_n_ctx(Some(unfittable_extra_n_ctx)); - let mut extra = FitExtraModel { - model_path: &model_path_c, - model_params: extra_params.as_mut(), - context_params: &mut extra_context_params, - shares_model: true, - }; - - let result = params.as_mut().fit_params( - &model_path_c, - &mut context_params, - &mut margins, - 512, - Some(&mut extra), - llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, - ); - - let fit = result.map_err(|fit_error| anyhow!("fit_params failed: {fit_error:?}"))?; - assert!(fit.n_ctx > 0); - assert_ne!( - extra_context_params.n_ctx(), - Some(unfittable_extra_n_ctx), - "the llama.cpp fit must overwrite the extra model's context size" - ); - assert_eq!( - extra_context_params.context_params.n_ctx, - context_params.context_params.n_ctx - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn special_tokens_exist(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - - assert_ne!(model.token_bos(), Some(eos)); - assert!(model.is_eog_token(&SampledToken::Content(eos))); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 512, - n_ubatch = 64, - embeddings = true, -)] -fn model_without_a_bos_token_reports_none(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_eq!(fixture.model.token_bos(), None); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_nl_is_the_first_token_of_a_tokenized_newline(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let newline_tokens = model.str_to_token("\n", AddBos::Never, ParseSpecialTokens::Never)?; - - assert_eq!(model.token_nl(), newline_tokens.first().copied()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_eog_token_classifies_reasoning_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - assert!(model.is_eog_token(&SampledToken::Reasoning(eos))); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_eog_token_classifies_tool_call_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - assert!(model.is_eog_token(&SampledToken::ToolCall(eos))); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn is_eog_token_classifies_undeterminable_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let eos = model.token_eos().context("model declares an EOS token")?; - assert!(model.is_eog_token(&SampledToken::Undeterminable(eos))); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn decoder_only_model_has_no_decoder_start_token(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_eq!(fixture.model.decode_start_token(), None); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn chat_model_has_no_separator_token(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_eq!(fixture.model.token_sep(), None); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_attr_returns_attrs_for_bos(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let bos = model.token_bos().context("model declares a BOS token")?; - let attrs = model.token_attr(bos)?; - let bit_repr = format!("{:?}", *attrs); - assert!( - !bit_repr.is_empty(), - "token_attr(bos) must produce Debug output" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let tokens = model.str_to_token("hello world", AddBos::Never, ParseSpecialTokens::Always)?; - assert_ne!(tokens, Vec::::new()); - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let piece = - model.token_to_piece(&SampledToken::Content(tokens[0]), &mut decoder, false, None)?; - - assert_ne!(piece, ""); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_grows_buffer_when_initial_estimation_too_small( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let many_short_chars = "a b c d e f g h i j k l"; - let tokens = - fixture - .model - .str_to_token(many_short_chars, AddBos::Always, ParseSpecialTokens::Always)?; - - assert!( - tokens.len() > 8, - "expected regrow; got {} tokens", - tokens.len() - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_with_add_bos_never(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let tokens_with_bos = - model.str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; - let tokens_without_bos = - model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - - assert!(tokens_with_bos.len() >= tokens_without_bos.len()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn str_to_token_with_many_tokens_triggers_buffer_resize(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::fmt::Write; - - let many_numbers = (0..2000).fold(String::new(), |mut accumulator, number| { - let _ = write!(accumulator, "{number} "); - accumulator - }); - - let tokens = - fixture - .model - .str_to_token(&many_numbers, AddBos::Always, ParseSpecialTokens::Always)?; - - assert!(tokens.len() > many_numbers.len() / 2); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_bytes_returns_bytes_for_known_token(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let bytes = model.token_to_piece_bytes(tokens[0], 32, false, None)?; - - assert_ne!(bytes, Vec::::new()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_handles_large_token_requiring_buffer_resize( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - - for (token, _) in model.tokens(true).take(200) { - let result = model.token_to_piece(&SampledToken::Content(token), &mut decoder, true, None); - assert!(result.is_ok()); - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_bytes_insufficient_buffer_returns_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let result = model.token_to_piece_bytes(tokens[0], 1, false, None); - - assert!( - result - .unwrap_err() - .to_string() - .contains("Insufficient Buffer Space") - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_with_lstrip(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; - let result = model.token_to_piece( - &SampledToken::Content(tokens[0]), - &mut decoder, - false, - NonZeroU16::new(1), - ); - - assert!(result.is_ok()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_decodes_reasoning_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; - - let piece = model.token_to_piece( - &SampledToken::Reasoning(tokens[0]), - &mut decoder, - true, - None, - )?; - - assert_ne!(piece, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_decodes_tool_call_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; - - let piece = - model.token_to_piece(&SampledToken::ToolCall(tokens[0]), &mut decoder, true, None)?; - - assert_ne!(piece, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_decodes_undeterminable_variant(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut decoder = encoding_rs::UTF_8.new_decoder(); - let tokens = model.str_to_token("hi", AddBos::Never, ParseSpecialTokens::Always)?; - - let piece = model.token_to_piece( - &SampledToken::Undeterminable(tokens[0]), - &mut decoder, - true, - None, - )?; - - assert_ne!(piece, ""); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn tokens_iterator_produces_valid_entries(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mut count = 0; - - for (token, _piece_result) in model.tokens(false) { - assert!(token.0 >= 0); - count += 1; - - if count >= 100 { - break; - } - } - - assert_eq!(count, 100); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn n_vocab_matches_tokens_iterator_count(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let n_vocab = model.n_vocab(); - let count = model.tokens(false).count(); - - assert_eq!(count, usize::try_from(n_vocab)?); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128 -)] -fn debug_format_includes_struct_name_and_model_field(fixture: &LlamaFixture<'_>) -> Result<()> { - let formatted = format!("{:?}", fixture.model); - - assert!(formatted.contains("LlamaModel")); - assert!(formatted.contains("model")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_adds_a_bos_token(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(fixture.model.adds_bos_token()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_does_not_add_a_bos_token(fixture: &LlamaFixture<'_>) -> Result<()> { - assert!(!fixture.model.adds_bos_token()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_to_piece_bytes_rejects_tokens_outside_the_vocabulary( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let n_vocab = model.n_vocab(); - - assert_eq!( - [LlamaToken::new(-1), LlamaToken::new(n_vocab)] - .map(|token| model.token_to_piece_bytes(token, 8, true, None)), - [ - Err(TokenToStringError::TokenOutsideVocabulary( - TokenOutsideVocabulary { - token: LlamaToken::new(-1), - n_vocab, - } - )), - Err(TokenToStringError::TokenOutsideVocabulary( - TokenOutsideVocabulary { - token: LlamaToken::new(n_vocab), - n_vocab, - } - )), - ] - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn token_attr_rejects_tokens_outside_the_vocabulary(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let n_vocab = model.n_vocab(); - - assert_eq!( - model.token_attr(LlamaToken::new(n_vocab)).err(), - Some(TokenAttrError::TokenOutsideVocabulary( - TokenOutsideVocabulary { - token: LlamaToken::new(n_vocab), - n_vocab, - } - )) - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/model_loading_errors.rs b/llama-cpp-bindings-tests/tests/model_loading_errors.rs index e8b151a70..c706ed7d0 100644 --- a/llama-cpp-bindings-tests/tests/model_loading_errors.rs +++ b/llama-cpp-bindings-tests/tests/model_loading_errors.rs @@ -1,143 +1,102 @@ +use std::env; +use std::ffi::CString; +use std::ffi::OsStr; +use std::fs; +use std::os::unix::ffi::OsStrExt; use std::path::Path; use std::path::PathBuf; use anyhow::Result; -use llama_cpp_bindings::LlamaLoraAdapterInitError; -use llama_cpp_bindings::LlamaModelLoadError; + +use llama_cpp_bindings::error::llama_model_load_error::LlamaModelLoadError; use llama_cpp_bindings::model::LlamaModel; use llama_cpp_bindings::model::params::LlamaModelParams; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, )] -fn load_model_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { +fn load_model_with_a_missing_file_reports_it_not_found(fixture: &LlamaFixture<'_>) { let model_params = LlamaModelParams::default(); let result = LlamaModel::load_from_file(fixture.backend, "/nonexistent/model.gguf", &model_params); - assert!(matches!( - result.unwrap_err(), - LlamaModelLoadError::FileNotFound(path) if path == Path::new("/nonexistent/model.gguf"), - )); - Ok(()) + assert_eq!( + result.err(), + Some(LlamaModelLoadError::FileNotFound(PathBuf::from( + "/nonexistent/model.gguf" + ))) + ); } #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, )] -fn load_model_with_invalid_file_content_returns_unloadable_or_reported( +fn load_model_with_invalid_file_content_reports_it_unloadable( fixture: &LlamaFixture<'_>, ) -> Result<()> { let model_params = LlamaModelParams::default(); - let dummy_path = std::env::temp_dir().join("llama_test_invalid_model.gguf"); - std::fs::write(&dummy_path, b"not a valid gguf model file")?; + let dummy_path = env::temp_dir().join("llama_test_invalid_model.gguf"); + fs::write(&dummy_path, b"not a valid gguf model file")?; let result = LlamaModel::load_from_file(fixture.backend, &dummy_path, &model_params); - assert!(matches!( - result.unwrap_err(), - LlamaModelLoadError::Unloadable | LlamaModelLoadError::Reported { .. }, - )); - let _ = std::fs::remove_file(&dummy_path); + assert_eq!(result.err(), Some(LlamaModelLoadError::Unloadable)); + fs::remove_file(&dummy_path)?; Ok(()) } #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, )] -fn load_model_with_non_utf8_path_returns_path_to_str_error( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - +fn load_model_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) { let model_params = LlamaModelParams::default(); let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); let result = LlamaModel::load_from_file(fixture.backend, non_utf8_path, &model_params); - assert!(matches!( - result.unwrap_err(), - LlamaModelLoadError::PathToStrError(path) if path == non_utf8_path.to_path_buf() - )); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn lora_adapter_init_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let result = fixture - .model - .lora_adapter_init("/nonexistent/path/lora.gguf"); assert_eq!( - result.unwrap_err(), - LlamaLoraAdapterInitError::FileNotFound(PathBuf::from("/nonexistent/path/lora.gguf")) + result.err(), + Some(LlamaModelLoadError::PathToStrError( + non_utf8_path.to_path_buf() + )) ); - Ok(()) } #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, )] -fn lora_adapter_init_with_invalid_gguf_returns_unloadable( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let dummy_path = std::env::temp_dir().join("llama_test_dummy_lora.gguf"); - std::fs::write(&dummy_path, b"not a valid gguf")?; - - let result = fixture.model.lora_adapter_init(&dummy_path); - - assert_eq!(result.unwrap_err(), LlamaLoraAdapterInitError::Unloadable); - let _ = std::fs::remove_file(&dummy_path); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn lora_adapter_init_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); - let result = fixture.model.lora_adapter_init(non_utf8_path); - +fn load_model_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) { assert_eq!( - result.unwrap_err(), - LlamaLoraAdapterInitError::PathToStrError(non_utf8_path.to_path_buf()) + LlamaModel::load_from_file( + fixture.backend, + "/models/model\0.gguf", + &LlamaModelParams::default() + ) + .err(), + CString::new("/models/model\0.gguf") + .err() + .map(LlamaModelLoadError::NullError) ); - Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/model_metadata.rs b/llama-cpp-bindings-tests/tests/model_metadata.rs new file mode 100644 index 000000000..790f051b0 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/model_metadata.rs @@ -0,0 +1,328 @@ +use std::ffi::CString; + +use anyhow::Result; + +use llama_cpp_bindings::error::meta_val_error::MetaValError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const INITIAL_META_VALUE_BUFFER_BYTES: usize = 256; +const OUT_OF_RANGE_META_INDEX_RETURN: i32 = -1; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) { + assert!(fixture.model.meta_count() > 0); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn the_first_metadata_entry_has_a_nonempty_key(fixture: &LlamaFixture<'_>) -> Result<()> { + let key = fixture.model.meta_key_by_index(0)?; + assert_ne!(key, ""); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn the_first_metadata_entry_has_a_nonempty_value(fixture: &LlamaFixture<'_>) -> Result<()> { + let value = fixture.model.meta_val_str_by_index(0)?; + assert_ne!(value, ""); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn meta_key_by_index_out_of_range_reports_the_negative_llama_cpp_return( + fixture: &LlamaFixture<'_>, +) { + assert_eq!( + fixture.model.meta_key_by_index(999_999), + Err(MetaValError::NegativeReturn(OUT_OF_RANGE_META_INDEX_RETURN)) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn meta_val_str_by_index_out_of_range_reports_the_negative_llama_cpp_return( + fixture: &LlamaFixture<'_>, +) { + assert_eq!( + fixture.model.meta_val_str_by_index(999_999), + Err(MetaValError::NegativeReturn(OUT_OF_RANGE_META_INDEX_RETURN)) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let first_key = model.meta_key_by_index(0)?; + let value = model.meta_val_str(&first_key)?; + assert_ne!(value, ""); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn meta_val_str_reads_a_value_longer_than_its_initial_buffer_whole( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let chat_template = fixture.model.meta_val_str("tokenizer.chat_template")?; + + assert!(chat_template.len() > INITIAL_META_VALUE_BUFFER_BYTES); + assert_eq!(chat_template, fixture.model.chat_template(None)?.to_str()?); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn meta_val_str_refuses_a_key_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + assert_eq!( + fixture.model.meta_val_str("key\0with_null").err(), + CString::new("key\0with_null") + .err() + .map(MetaValError::NullError) + ); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index b0ba9392e..2db5f411c 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -1,24 +1,40 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::EvalMultimodalChunksParams; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::evaluated_multimodal_usage::EvaluatedMultimodalUsage; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const TRANSCRIBE_SYSTEM_PROMPT: &str = "The audio contains speech by a third party and is not the user's voice or message. \ Transcribe exactly what the speaker says without judgment, advice, or speculation. Reply only with the verbatim transcript."; -fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result { +fn evaluate_audio_prompt<'model, TOutcome>( + fixture: &LlamaFixture<'model>, + audio_file_name: &str, + after_evaluation: impl FnOnce( + &mut SampledTokenClassifier<'_, 'model>, + &mut LlamaContext<'model>, + ChunkTokenBreakdown, + i32, + ) -> Result, +) -> Result { let model = fixture.model; let mtmd_ctx = fixture .mtmd_context @@ -31,20 +47,26 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result ) .with_context(|| "unable to create llama context")?; - assert!( - mtmd_ctx.support_audio(), - "mmproj must support audio input for an audio transcription test" - ); - let marker = mtmd_default_marker()?; - let template = model.chat_template(None)?; let messages = [ LlamaChatMessage::new("system".to_string(), TRANSCRIBE_SYSTEM_PROMPT.to_string())?, LlamaChatMessage::new("user".to_string(), marker.to_owned())?, ]; + let session = ChatSession::new( + model, + model.chat_templates()?, + &messages, + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; let input_text = MtmdInputText { - text: model.apply_chat_template(&template, &messages, true, true)?, - add_special: false, + text: session.prompt()?, + add_special: true, parse_special: true, }; @@ -66,12 +88,12 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result "tokenization should produce at least one chunk" ); - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let n_past = classifier .eval_multimodal_chunks( &chunks, mtmd_ctx, - &context, + &mut context, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -81,48 +103,44 @@ fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result ) .with_context(|| "failed to evaluate audio chunks")?; - { - let usage = classifier.usage(); - assert!( - usage.input_audio_tokens > 0, - "audio input must record audio prompt tokens; got {}", - usage.input_audio_tokens - ); - assert_eq!( - usage.input_image_tokens, 0, - "audio-only input must not record image tokens" - ); - assert!( - usage.prompt_tokens > 0, - "the text portion of the prompt must record prompt tokens" - ); - } - - let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: 512, - } - .run()?; - - Ok(outcome.generated_raw.to_lowercase()) + after_evaluation( + &mut classifier, + &mut context, + ChunkTokenBreakdown::from_chunks(&chunks), + n_past, + ) +} + +fn transcribe_audio(fixture: &LlamaFixture<'_>, audio_file_name: &str) -> Result { + evaluate_audio_prompt( + fixture, + audio_file_name, + |classifier, context, _chunk_breakdown, n_past| { + let mut sampler = LlamaSampler::greedy()?; + let mut batch = LlamaBatch::new(ContextLimits::of(context)); + let outcome = ClassifySampleLoop { + classifier, + sampler: &mut sampler, + context, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: 512, + } + .run(); + + Ok(outcome.generated.text().to_owned()) + }, + ) } -fn assert_audio_transcription_contains( +fn assert_audio_transcription( fixture: &LlamaFixture<'_>, audio_file_name: &str, - expected_word: &str, + expected_transcript: &str, ) -> Result<()> { - let transcript = transcribe_audio(fixture, audio_file_name)?; - - assert!( - transcript.contains(expected_word), - "transcription should echo the spoken word {expected_word:?}; got: {transcript:?}" + assert_eq!( + transcribe_audio(fixture, audio_file_name)?, + expected_transcript ); Ok(()) @@ -133,7 +151,7 @@ fn assert_audio_transcription_contains( "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "Llama-3.2-1B-Instruct-Q4_K_M.gguf" ), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, @@ -143,13 +161,14 @@ fn assert_audio_transcription_contains( "mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf" ), )] -fn audio_reaches_the_model_and_produces_a_response(fixture: &LlamaFixture<'_>) -> Result<()> { - let response = transcribe_audio(fixture, "quick_brown_fox.wav")?; - - assert!( - !response.is_empty(), - "the audio must reach the decoder and produce output; this model is too small to \ - follow the transcription instruction reliably, so only the pipeline is asserted here" +fn ultravox_transcribes_the_spoken_sentence_after_a_preamble( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + assert_eq!( + transcribe_audio(fixture, "quick_brown_fox.wav")?, + "I can provide a transcript of the audio. However, please note that the audio itself is \ + not provided. Based on the context, I'll attempt to create a verbatim transcript.\n\nThe \ + quick brown fox jumps over the lazy dog." ); Ok(()) @@ -157,52 +176,82 @@ fn audio_reaches_the_model_and_produces_a_response(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn audio_mmproj_reports_audio_support(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - assert!( - mtmd_ctx.support_audio(), - "an audio mmproj must report audio support" - ); - assert!( - mtmd_ctx.get_audio_sample_rate().is_some(), - "an audio-capable mmproj must report a required sample rate" - ); - - Ok(()) +fn gemma4_transcribes_the_quick_brown_fox_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_audio_transcription( + fixture, + "quick_brown_fox.wav", + "The quick brown fox jumps over the lazy dog.", + ) } #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn audio_transcribes_spoken_word(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_audio_transcription_contains(fixture, "quick_brown_fox.wav", "fox") +fn gemma4_transcribes_the_orange_cat_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_audio_transcription( + fixture, + "orange_cat.wav", + "The orange cat sat on the wooden fence.", + ) } +#[llama_test( + model_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "Llama-3.2-1B-Instruct-Q4_K_M.gguf" + ), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf" + ), +)] #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn audio_transcribes_uncommon_sentence(fixture: &LlamaFixture<'_>) -> Result<()> { - assert_audio_transcription_contains(fixture, "orange_cat.wav", "fence") +fn audio_input_records_each_chunk_type_in_its_own_usage_counter( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let EvaluatedMultimodalUsage { + usage, + chunk_breakdown, + } = evaluate_audio_prompt( + fixture, + "quick_brown_fox.wav", + |classifier, _context, chunk_breakdown, _n_past| { + Ok(EvaluatedMultimodalUsage { + usage: *classifier.usage(), + chunk_breakdown, + }) + }, + )?; + + assert_eq!(usage.input_audio_tokens, chunk_breakdown.audio); + assert_eq!(usage.input_image_tokens, chunk_breakdown.image); + assert_eq!(usage.prompt_tokens, chunk_breakdown.text); + + Ok(()) } diff --git a/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs new file mode 100644 index 000000000..dd5155f05 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_bitmap_construction.rs @@ -0,0 +1,81 @@ +use std::ffi::CString; +use std::fs; + +use anyhow::Result; + +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_bitmap_error::MtmdBitmapError; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const LLAMAS_JPG_WIDTH: u32 = 640; +const LLAMAS_JPG_HEIGHT: u32 = 427; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn from_buffer_decodes_image_bytes_into_an_image_bitmap_of_their_size( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let fixtures = fixtures_dir(); + let image_path = fixtures.join("llamas.jpg"); + let image_bytes = fs::read(&image_path)?; + let bitmap = MtmdBitmap::from_buffer(mtmd_ctx, &image_bytes)?; + + assert_eq!(bitmap.nx(), LLAMAS_JPG_WIDTH); + assert_eq!(bitmap.ny(), LLAMAS_JPG_HEIGHT); + assert!(!bitmap.is_audio()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn from_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + assert_eq!( + MtmdBitmap::from_file(mtmd_ctx, "path\0null").err(), + CString::new("path\0null") + .err() + .map(MtmdBitmapError::CStringError) + ); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs new file mode 100644 index 000000000..8beb3de5e --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_copy.rs @@ -0,0 +1,46 @@ +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn copy_creates_owned_duplicate(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let input_text = MtmdInputText { + text: "Hello <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let first_chunk = chunks.get(0).context("missing first chunk")?; + let copied = first_chunk.copy()?; + + assert!(copied.owned); + assert_eq!(copied.n_tokens(), first_chunk.n_tokens()); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs new file mode 100644 index 000000000..fad59e28e --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_encoding.rs @@ -0,0 +1,48 @@ +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn encode_chunk_encodes_an_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let input_text = MtmdInputText { + text: "Describe: <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let image_chunk = (0..chunks.len()) + .filter_map(|chunk_index| chunks.get(chunk_index)) + .find(|chunk| chunk.chunk_type() == Ok(MtmdInputChunkType::Image)) + .context("a prompt with an image must tokenize into an image chunk")?; + + assert_eq!(mtmd_ctx.encode_chunk(&image_chunk), Ok(())); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs new file mode 100644 index 000000000..08658cbcf --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_evaluation.rs @@ -0,0 +1,1007 @@ +use anyhow::Context; +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_attention_type::LlamaAttentionType; +use llama_cpp_bindings::error::eval_multimodal_chunks_error::EvalMultimodalChunksError; +use llama_cpp_bindings::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunk::FittedMtmdInputChunk; +use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunks::FittedMtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_eval_error::MtmdEvalError; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_bindings::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; +use llama_cpp_bindings_tests::prime_memory_with::prime_memory_with; +use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +struct ImageDimensions { + width: u32, + height: u32, +} + +const EVALUATED_IMAGE_DIMENSIONS: [ImageDimensions; 10] = [ + ImageDimensions { + width: 224, + height: 224, + }, + ImageDimensions { + width: 512, + height: 512, + }, + ImageDimensions { + width: 100, + height: 500, + }, + ImageDimensions { + width: 337, + height: 421, + }, + ImageDimensions { + width: 2, + height: 2, + }, + ImageDimensions { + width: 7, + height: 13, + }, + ImageDimensions { + width: 3, + height: 1000, + }, + ImageDimensions { + width: 1000, + height: 3, + }, + ImageDimensions { + width: 1920, + height: 1080, + }, + ImageDimensions { + width: 4096, + height: 4096, + }, +]; + +fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; (width as usize) * (height as usize) * 3]; + let bitmap = MtmdBitmap::from_image_data(width, height, &image_data)?; + let input_text = MtmdInputText { + text: "Describe: <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let n_positions = chunks.total_positions(); + let required_n_ctx = u32::try_from(n_positions + 256)?; + if fixture.context_params.n_ctx < required_n_ctx { + bail!( + "fixture n_ctx ({}) below required ({}) for {}x{} image", + fixture.context_params.n_ctx, + required_n_ctx, + width, + height, + ); + } + + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false)?, + n_positions, + "evaluating a {width}x{height} image must advance past every chunk position" + ); + Ok(()) +} + +fn tokenize_question_about_llamas( + fixture: &LlamaFixture<'_>, + mtmd_ctx: &MtmdContext, +) -> Result { + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .with_context(|| "image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + Ok(mtmd_ctx.tokenize( + MtmdInputText { + text: media_question_chat_session(fixture.model, PROMPT_QUESTION)?.prompt()?, + add_special: true, + parse_special: true, + }, + &[&bitmap], + )?) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 64, + n_batch = 64, + n_ubatch = 32, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 64, + n_batch = 64, + n_ubatch = 32, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let chunks = MtmdInputChunks::new()?; + let huge_batch = i32::try_from(llama_ctx.n_batch() + 1)?; + + let result = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, huge_batch, false); + + assert_eq!( + result, + Err(MtmdEvalError::BatchSizeExceedsContextLimit { + requested: huge_batch, + context_max: llama_ctx.n_batch(), + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_reports_an_exception_llama_cpp_throws_while_evaluating_a_chunk( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); + let mut llama_ctx = fixture.build_context()?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_EVAL_CHUNK_SINGLE_THROWS); + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, true), + Err(MtmdEvalError::Reported { + message: injected_fault_message() + }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_reports_a_chunk_llama_cpp_does_not_return_before_evaluating_any( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); + let mut llama_ctx = fixture.build_context()?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let evaluation = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL, + ); + + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, true) + }; + + assert_eq!( + evaluation, + Err(MtmdEvalError::FfiContract(FfiContractError { + operation: "mtmd_input_chunks_get", + detail: "returned a null chunk within the chunk count", + })) + ); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), + Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) + ); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn fitting_no_chunks_leaves_nothing_to_evaluate(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_chunks = + FittedMtmdInputChunks::fit(MtmdInputChunks::new()?, mtmd_ctx, llama_ctx.n_ubatch())?; + + assert!(fitted_chunks.is_empty()); + assert!(FittedMtmdInputChunk::at(&fitted_chunks, 0).is_none()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_chunks = FittedMtmdInputChunks::fit( + tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"), + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + let first_chunk = + FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; + + assert_eq!( + first_chunk.eval_single(mtmd_ctx, &mut llama_ctx, 0, 0, 0, true), + Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) + ); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_single_rejects_a_batch_larger_than_the_context_batch_before_evaluating( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = fixture.build_context()?; + let fitted_chunks = FittedMtmdInputChunks::fit( + tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"), + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + let first_chunk = + FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; + let context_max = llama_ctx.n_batch(); + let oversized_batch = i32::try_from(context_max + 1)?; + + assert_eq!( + first_chunk.eval_single(mtmd_ctx, &mut llama_ctx, 0, 0, oversized_batch, true), + Err(MtmdEvalError::BatchSizeExceedsContextLimit { + requested: oversized_batch, + context_max, + }) + ); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_rejects_a_sequence_the_context_does_not_have_instead_of_aborting( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = fixture.build_context()?; + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); + let n_seq_max = llama_ctx.n_seq_max(); + let missing_sequence = i32::try_from(n_seq_max)?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, missing_sequence, n_batch, true), + Err(MtmdEvalError::SequenceIdOutOfRange(SequenceIdOutOfRange { + seq_id: missing_sequence, + n_seq_max, + })) + ); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluating( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_micro_batch_tokens = llama_ctx.n_ubatch(); + let fitted_chunks = FittedMtmdInputChunks::fit( + tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"), + mtmd_ctx, + fitted_micro_batch_tokens, + )?; + let first_chunk = + FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; + let smaller_micro_batch_tokens = fitted_micro_batch_tokens - 1; + + assert_eq!( + first_chunk.eval_single( + mtmd_ctx, + &mut llama_ctx, + 0, + 0, + i32::try_from(smaller_micro_batch_tokens)?, + true + ), + Err(MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted { + evaluated_micro_batch_tokens: smaller_micro_batch_tokens, + fitted_micro_batch_tokens, + }) + ); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Describe: <__media__>"); + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + assert!(matches!( + classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut llama_ctx, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 0, + logits_last: true, + }, + ), + Err(EvalMultimodalChunksError::EvalFailed( + MtmdEvalError::NonPositiveBatchSize { requested: 0 } + )) + )); + assert_eq!(classifier.usage().prompt_tokens, 0); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_advances_past_every_position_of_a_standard_image( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let fixtures = fixtures_dir(); + let image_path = fixtures.join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .context("image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + let input_text = MtmdInputText { + text: "What is in this image? <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let n_positions = chunks.total_positions(); + let required_n_ctx = u32::try_from(n_positions + 256)?; + assert!( + fixture.context_params.n_ctx >= required_n_ctx, + "fixture n_ctx ({}) below required ({}); update the attribute literal", + fixture.context_params.n_ctx, + required_n_ctx, + ); + + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false), + Ok(n_positions) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_evaluates_images_of_ordinary_and_extreme_dimensions( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + for ImageDimensions { width, height } in EVALUATED_IMAGE_DIMENSIONS { + eval_synthetic_bitmap(fixture, width, height) + .context(format!("dimension {width}x{height} should succeed"))?; + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 2048, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma3_decoding_follows_the_causal_attention_the_mtmd_helper_leaves_on_the_context( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params) + .into_llama_context_params() + .with_attention_type(LlamaAttentionType::NonCausal), + )?; + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let next_position = chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, false)?; + let tokens = fixture.model.str_to_token( + &"llama ".repeat(600), + AddBos::Never, + ParseSpecialTokens::Never, + )?; + let last_index = tokens.len() - 1; + let mut batch = LlamaBatch::new(ContextLimits::of(&llama_ctx)); + + for (index, token) in tokens.iter().enumerate() { + batch.add( + &SampledToken::Content(*token), + next_position + i32::try_from(index)?, + &[0], + index == last_index, + )?; + } + + assert!(batch.n_tokens().unsigned_abs() > llama_ctx.n_ubatch()); + assert_eq!(llama_ctx.decode(&batch), Ok(())); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 2048, + n_ubatch = 128, + mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma3_eval_chunks_rejects_a_non_causal_image_larger_than_the_micro_batch_before_evaluating( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let image_tokens = ChunkTokenBreakdown::from_chunks(&chunks).image; + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let micro_batch_tokens = llama_ctx.n_ubatch(); + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, 0, 0, n_batch, true), + Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( + NonCausalChunkMicroBatchMismatch { + chunk_tokens: usize::try_from(image_tokens)?, + micro_batch_tokens, + } + )) + ); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 2048, + n_ubatch = 128, + mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma3_fitting_rejects_a_non_causal_image_larger_than_the_micro_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let image_chunk = (0..chunks.len()) + .filter_map(|chunk_index| chunks.get(chunk_index)) + .find(|chunk| chunk.chunk_type() == Ok(MtmdInputChunkType::Image)) + .context("the prompt must contain an image chunk")?; + let llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + assert_eq!( + image_chunk.fit_to_micro_batch(mtmd_ctx, llama_ctx.n_ubatch()), + Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( + NonCausalChunkMicroBatchMismatch { + chunk_tokens: image_chunk.n_tokens(), + micro_batch_tokens: llama_ctx.n_ubatch(), + } + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 2048, + n_ubatch = 128, + mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma3_classifier_leaves_the_prompt_untouched_when_a_non_causal_image_exceeds_the_micro_batch( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + let n_batch = i32::try_from(llama_ctx.n_batch())?; + let evaluation = classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut llama_ctx, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch, + logits_last: true, + }, + ); + + assert!(matches!( + evaluation, + Err(EvalMultimodalChunksError::EvalFailed( + MtmdEvalError::NonCausalChunkExceedsMicroBatch(NonCausalChunkMicroBatchMismatch { + micro_batch_tokens, + .. + }) + )) if micro_batch_tokens == llama_ctx.n_ubatch() + )); + assert_eq!(classifier.usage().prompt_tokens, 0); + assert_eq!(llama_ctx.memory_seq_pos_max(0)?, -1); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn classifier_reports_an_image_the_context_has_no_room_left_for( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let breakdown = ChunkTokenBreakdown::from_chunks(&chunks); + let mut llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + prime_memory_with(fixture, &mut llama_ctx, &"hello ".repeat(230))?; + + let primed_positions = llama_ctx.memory_seq_pos_max(0)? + 1; + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + assert!( + u64::from(llama_ctx.n_ctx()) - u64::try_from(primed_positions)? < breakdown.image, + "the primed context must have less room left than the image needs" + ); + assert!(matches!( + classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut llama_ctx, + EvalMultimodalChunksParams { + start_position: primed_positions, + seq_id: 0, + n_batch: 256, + logits_last: true, + }, + ), + Err(EvalMultimodalChunksError::EvalFailed( + MtmdEvalError::EvalFailed { code: 1 } + )) + )); + assert_eq!(classifier.usage().input_image_tokens, 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_chunks_reports_an_image_the_context_has_no_room_left_for( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let mut llama_ctx = fixture.build_context()?; + + prime_memory_with(fixture, &mut llama_ctx, &"hello ".repeat(230))?; + + let primed_positions = llama_ctx.memory_seq_pos_max(0)? + 1; + + assert_eq!( + chunks.eval_chunks(mtmd_ctx, &mut llama_ctx, primed_positions, 0, 256, true), + Err(MtmdEvalError::EvalFailed { code: 1 }) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn fitting_reports_a_chunk_llama_cpp_does_not_return(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL, + ); + + assert_eq!( + chunks.fit_to_micro_batch(mtmd_ctx, 256), + Err(MtmdEvalError::FfiContract(FfiContractError { + operation: "mtmd_input_chunks_get", + detail: "returned a null chunk within the chunk count", + })) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn fitting_reports_a_chunk_type_the_bindings_do_not_recognize( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNK_TYPE_UNKNOWN); + + assert_eq!( + chunks.fit_to_micro_batch(mtmd_ctx, 256), + Err(MtmdEvalError::UnknownChunkType(MtmdInputChunkTypeError( + llama_cpp_bindings_sys::MTMD_INPUT_CHUNK_TYPE_COUNT + ))) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 256, + n_ubatch = 256, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn classifier_reports_a_fitted_chunk_llama_cpp_no_longer_returns( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; + let mut llama_ctx = fixture.build_context()?; + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + let _fault = ArmedFault::after_passing( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_GET_RETURNS_NULL, + chunks.len(), + ); + + assert_eq!( + classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut llama_ctx, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 256, + logits_last: true, + }, + ), + Err(EvalMultimodalChunksError::ChunkOutOfBounds(0)) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs new file mode 100644 index 000000000..c5b944f0b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_chunk_tokenization.rs @@ -0,0 +1,359 @@ +use std::ffi::CString; + +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_input_chunk::MtmdInputChunk; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_bindings::mtmd::mtmd_tokenize_error::MtmdTokenizeError; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_tests::tokenize_prompt_with_synthetic_image::tokenize_prompt_with_synthetic_image; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +fn first_image_chunk(chunks: &MtmdInputChunks) -> Result { + (0..chunks.len()) + .filter_map(|chunk_index| chunks.get(chunk_index)) + .find(|chunk| chunk.chunk_type() == Ok(MtmdInputChunkType::Image)) + .context("a prompt with an image must tokenize into an image chunk") +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello world <__media__>"); + let first_chunk = chunks.get(0).context("missing first chunk")?; + assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello world <__media__>"); + let first_chunk = chunks.get(0).context("missing first chunk")?; + let tokens = first_chunk + .text_tokens_of(first_chunk.chunk_type()?) + .context("the first chunk of a text prompt must carry tokens")?; + + assert_ne!(tokens, Vec::::new()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello world <__media__>"); + let first_chunk = chunks.get(0).context("missing first chunk")?; + assert!(first_chunk.n_tokens() > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello world <__media__>"); + let first_chunk = chunks.get(0).context("missing first chunk")?; + assert!(first_chunk.n_positions() > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); + let first_chunk = chunks.get(0).context("missing first chunk")?; + assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); + assert_eq!(first_chunk.id(), Ok(None)); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn an_image_chunk_has_no_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); + + assert_eq!( + first_image_chunk(&chunks)?.text_tokens_of(MtmdInputChunkType::Image), + None + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn an_image_chunk_carries_the_empty_id_of_a_bitmap_without_one( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let chunks = tokenize_prompt_with_synthetic_image(fixture, "Hello <__media__>"); + + assert_eq!(first_image_chunk(&chunks)?.id(), Ok(Some(String::new()))); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenizing_text_with_an_image_produces_chunks(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let input_text = MtmdInputText { + text: "Describe this image: <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + assert!(!chunks.is_empty()); + assert!(chunks.total_tokens() > 0); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenizing_more_bitmaps_than_media_markers_reports_a_tokenization_failure( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let input_text = MtmdInputText { + text: "No media markers here".to_string(), + add_special: true, + parse_special: true, + }; + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + + assert_eq!( + mtmd_ctx.tokenize(input_text, &[&bitmap]).err(), + Some(MtmdTokenizeError::MediaPreprocessingOrTokenizationFailed) + ); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenize_refuses_text_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let input_text = MtmdInputText { + text: "text\0null".to_string(), + add_special: true, + parse_special: true, + }; + + assert_eq!( + mtmd_ctx.tokenize(input_text, &[]).err(), + CString::new("text\0null") + .err() + .map(MtmdTokenizeError::CStringError) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenizing_reports_a_chunk_list_llama_cpp_cannot_allocate(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_INIT_THROWS); + + assert_eq!( + mtmd_ctx + .tokenize( + MtmdInputText { + text: "Hello".to_owned(), + add_special: true, + parse_special: true, + }, + &[], + ) + .err(), + Some(MtmdTokenizeError::InputChunksError( + MtmdInputChunksError::Reported { + message: injected_fault_message(), + } + )) + ); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs index d82c8ac14..d402d824d 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_fusion.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_fusion.rs @@ -1,71 +1,77 @@ use anyhow::Context; use anyhow::Result; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::EvalMultimodalChunksParams; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::LlamaChatMessage; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdContext; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::mtmd_default_marker; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -const RECOGNIZED_ANIMALS: [&str; 3] = ["llama", "alpaca", "sheep"]; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const MAX_GENERATED_TOKENS: i32 = 512; const DESCRIBE_INSTRUCTION: &str = "Describe the animal shown in the image, then write the exact words spoken in the audio."; -fn build_describe_image_and_audio_prompt(model: &LlamaModel) -> Result { +fn describe_image_and_audio_chat_session(model: &LlamaModel) -> Result> { let marker = mtmd_default_marker()?; - let template = model.chat_template(None)?; let user_content = format!("Image: {marker}\nAudio: {marker}\n{DESCRIBE_INSTRUCTION}"); let messages = [LlamaChatMessage::new("user".to_string(), user_content)?]; - Ok(model.apply_chat_template(&template, &messages, true, false)?) + Ok(ChatSession::new( + model, + model.chat_templates()?, + &messages, + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?) } fn load_fixture_bitmap(mtmd_ctx: &MtmdContext, file_name: &str) -> Result { let path = fixtures_dir().join(file_name); let path_str = path .to_str() - .with_context(|| format!("{file_name} path is not valid UTF-8"))?; + .context(format!("{file_name} path is not valid UTF-8"))?; MtmdBitmap::from_file(mtmd_ctx, path_str) - .with_context(|| format!("failed to load {file_name} from file")) + .context(format!("failed to load {file_name} from file")) } #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 4096, n_batch = 512, n_ubatch = 512, mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), )] -fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { +fn an_image_and_audio_prompt_is_answered_with_both_media_counted_as_input( + fixture: &LlamaFixture<'_>, +) -> Result<()> { let model = fixture.model; let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); - assert!( - mtmd_ctx.support_vision(), - "mmproj must support vision input for a combined image and audio test" - ); - assert!( - mtmd_ctx.support_audio(), - "mmproj must support audio input for a combined image and audio test" - ); - let image_bitmap = load_fixture_bitmap(mtmd_ctx, "llamas.jpg")?; assert!(!image_bitmap.is_audio(), "llamas.jpg must decode as image"); @@ -75,9 +81,10 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { "orange_cat.wav must decode as audio" ); + let session = describe_image_and_audio_chat_session(model)?; let input_text = MtmdInputText { - text: build_describe_image_and_audio_prompt(model)?, - add_special: false, + text: session.prompt()?, + add_special: true, parse_special: true, }; @@ -85,7 +92,7 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { .tokenize(input_text, &[&image_bitmap, &audio_bitmap]) .with_context(|| "failed to tokenize combined image and audio input")?; - let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; + let expected = ChunkTokenBreakdown::from_chunks(&chunks); assert!( expected.image > 0, "image input must produce at least one image-chunk token" @@ -111,12 +118,12 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { .with_context(|| "unable to create llama context")?; let n_batch = i32::try_from(context.n_batch())?; - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); let n_past = classifier .eval_multimodal_chunks( &chunks, mtmd_ctx, - &context, + &mut context, EvalMultimodalChunksParams { start_position: 0, seq_id: 0, @@ -134,7 +141,7 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { } let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); let outcome = ClassifySampleLoop { classifier: &mut classifier, sampler: &mut sampler, @@ -143,24 +150,12 @@ fn image_and_audio_together(fixture: &LlamaFixture<'_>) -> Result<()> { initial_position: n_past, max_generated_tokens: MAX_GENERATED_TOKENS, } - .run()?; + .run(); - let description = outcome.generated_raw.to_lowercase(); - assert!( - !description.is_empty(), - "model should generate a description from combined image and audio input" - ); - assert!( - RECOGNIZED_ANIMALS - .iter() - .any(|animal| description.contains(animal)), - "the description must name one of the animals a vision encoder plausibly reports for \ - llamas.jpg ({RECOGNIZED_ANIMALS:?}), which is what proves the image reached the \ - output; got: {description:?}" - ); - assert!( - description.contains("fence"), - "description should echo the spoken word \"fence\" from the audio; got: {description:?}" + assert!(outcome.eog_seen, "{outcome:?}"); + assert_eq!( + outcome.generated.text(), + "Animal: The image shows several sheep.\n\nAudio: The orange cat sat on the wooden fence." ); Ok(()) diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs new file mode 100644 index 000000000..8b8d7f359 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_capabilities.rs @@ -0,0 +1,203 @@ +use anyhow::Context as _; +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_capability::MtmdCapability; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const AUDIO_SAMPLE_RATE_HZ: u32 = 16_000; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn qwen_image_chunks_decode_causally(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let image_data = vec![128u8; 64 * 64 * 3]; + let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; + let input_text = MtmdInputText { + text: "Describe: <__media__>".to_string(), + add_special: true, + parse_special: true, + }; + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + for chunk_index in 0..chunks.len() { + let chunk = chunks + .get(chunk_index) + .context(format!("missing chunk at index {chunk_index}"))?; + if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { + assert!(!mtmd_ctx.decode_use_non_causal(&chunk)); + + return Ok(()); + } + } + bail!("tokenization should produce at least one Image chunk"); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn qwen_mmproj_reports_vision_only_chat_capabilities(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + assert_eq!( + mtmd_ctx.capabilities(fixture.model), + MtmdCapability::Chat | MtmdCapability::Vision | MtmdCapability::TextGeneration + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn decode_use_mrope_is_true_for_qwen_vision(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + assert!( + mtmd_ctx.decode_use_mrope(), + "Qwen 3.5 / 3.6 mmproj uses mrope; decode_use_mrope must return true" + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn get_audio_sample_rate_is_none_for_vision_only_mmproj(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + assert!( + mtmd_ctx.get_audio_sample_rate().is_none(), + "Qwen 3.5 / 3.6 mmproj has no audio; get_audio_sample_rate must return None" + ); +} + +#[llama_test( + model_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "Llama-3.2-1B-Instruct-Q4_K_M.gguf" + ), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf" + ), +)] +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), +)] +fn audio_mmproj_reports_audio_support_at_sixteen_kilohertz(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + assert!( + mtmd_ctx + .capabilities(fixture.model) + .contains(MtmdCapability::Audio), + "an audio mmproj must report audio support" + ); + assert_eq!(mtmd_ctx.get_audio_sample_rate(), Some(AUDIO_SAMPLE_RATE_HZ)); +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma4_mmproj_reports_chat_vision_audio_and_text_generation_capabilities( + fixture: &LlamaFixture<'_>, +) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + assert_eq!( + mtmd_ctx.capabilities(fixture.model), + MtmdCapability::Chat + | MtmdCapability::Vision + | MtmdCapability::Audio + | MtmdCapability::TextGeneration + ); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs new file mode 100644 index 000000000..e9c43dbd5 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_projector_loading.rs @@ -0,0 +1,66 @@ +use std::ffi::CString; +use std::num::NonZeroU32; + +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; +use llama_cpp_bindings::mtmd::mtmd_context_params_error::MtmdContextParamsError; +use llama_cpp_bindings::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; +use llama_cpp_bindings::mtmd::mtmd_init_error::MtmdInitError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn init_from_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) { + let mtmd_params = MtmdContextParams::default(); + + assert_eq!( + MtmdContext::init_from_file("path\0null", fixture.model, &mtmd_params).err(), + CString::new("path\0null") + .err() + .map(MtmdInitError::CStringError) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn init_from_file_refuses_an_image_token_limit_a_c_int_cannot_hold(fixture: &LlamaFixture<'_>) { + let limit = MtmdImageTokenLimit::Tokens(NonZeroU32::MAX); + let mtmd_params = MtmdContextParams { + image_max_tokens: limit, + ..MtmdContextParams::default() + }; + + assert_eq!( + MtmdContext::init_from_file("mmproj.gguf", fixture.model, &mtmd_params).err(), + i32::try_from(u32::MAX).err().map(|source| { + MtmdInitError::ContextParams(MtmdContextParamsError::ImageMaxTokensExceedCInt { + limit, + source, + }) + }) + ); +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs index 303883059..0d850a930 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_prompt_parts_tokenization.rs @@ -1,19 +1,23 @@ use std::ffi::CString; +use anyhow::Context as _; use anyhow::Result; -use anyhow::anyhow; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdInputPart; -use llama_cpp_bindings::mtmd::MtmdInputTextPart; -use llama_cpp_bindings::mtmd::MtmdTokenizeError; -use llama_cpp_bindings::mtmd::mtmd_default_marker; + +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_chunks::MtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; +use llama_cpp_bindings::mtmd::mtmd_input_part::MtmdInputPart; +use llama_cpp_bindings::mtmd::mtmd_input_text_part::MtmdInputTextPart; +use llama_cpp_bindings::mtmd::mtmd_tokenize_error::MtmdTokenizeError; use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; const SPECIAL_TOKEN_TEXT: &str = "<|im_start|>"; @@ -30,7 +34,7 @@ fn tokens_of_text_chunks(chunks: &MtmdInputChunks) -> Result> { for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + .context(format!("missing chunk at index {chunk_index}"))?; if let Some(chunk_tokens) = chunk.text_tokens_of(chunk.chunk_type()?) { tokens.extend_from_slice(chunk_tokens); @@ -42,7 +46,7 @@ fn tokens_of_text_chunks(chunks: &MtmdInputChunks) -> Result> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -71,7 +75,7 @@ fn only_bitmap_parts_become_image_chunks(fixture: &LlamaFixture<'_>) -> Result<( for chunk_index in 0..chunks.len() { let chunk = chunks .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; + .context(format!("missing chunk at index {chunk_index}"))?; if chunk.chunk_type()? == MtmdInputChunkType::Image { image_chunks += 1; @@ -85,7 +89,7 @@ fn only_bitmap_parts_become_image_chunks(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -113,7 +117,7 @@ fn a_media_marker_inside_a_text_part_tokenizes_as_plain_text( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -154,14 +158,14 @@ fn parse_special_applies_to_each_text_part_on_its_own(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), )] -fn a_text_part_containing_a_nul_byte_is_rejected(fixture: &LlamaFixture<'_>) -> Result<()> { +fn a_text_part_containing_a_nul_byte_is_rejected(fixture: &LlamaFixture<'_>) { let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); @@ -175,6 +179,59 @@ fn a_text_part_containing_a_nul_byte_is_rejected(fixture: &LlamaFixture<'_>) -> .err() .map(MtmdTokenizeError::CStringError) ); +} - Ok(()) +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenizing_parts_reports_a_chunk_list_llama_cpp_cannot_allocate(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_INPUT_CHUNKS_INIT_THROWS); + + assert_eq!( + mtmd_ctx + .tokenize_from_parts(&[text_part("Hello", true)], true) + .err(), + Some(MtmdTokenizeError::InputChunksError( + MtmdInputChunksError::Reported { + message: injected_fault_message(), + } + )) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn tokenizing_parts_reports_a_tokenizer_llama_cpp_fails(fixture: &LlamaFixture<'_>) { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_TOKENIZE_FROM_PARTS_THROWS, + ); + + assert_eq!( + mtmd_ctx + .tokenize_from_parts(&[text_part("Hello", true)], true) + .err(), + Some(MtmdTokenizeError::Reported { + message: injected_fault_message(), + }) + ); } diff --git a/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs new file mode 100644 index 000000000..3e0fc4e9c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_reasoning_classification.rs @@ -0,0 +1,334 @@ +use anyhow::Context as _; +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::media_question_chat_session::media_question_chat_session; +use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), +)] +fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const MAX_GENERATED_TOKENS: i32 = 200; + + let model = fixture.model; + let backend = fixture.backend; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .context("image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let session = media_question_chat_session(model, PROMPT_QUESTION)?; + let input_text = MtmdInputText { + text: session.prompt()?, + add_special: true, + parse_special: true, + }; + + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let n_past = classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut context, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + )?; + + let mut sampler = LlamaSampler::chain([ + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, + LlamaSampler::top_k(40)?, + LlamaSampler::top_p(0.9, 1)?, + LlamaSampler::min_p(0.05, 1)?, + LlamaSampler::temp(0.7)?, + LlamaSampler::dist(0x00C0_FFEE)?, + ])?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: MAX_GENERATED_TOKENS, + } + .run(); + + let usage = classifier.usage(); + + if outcome.observed_reasoning == 0 { + bail!( + "Gemma 4 multimodal + thinking: classifier must emit at least one Reasoning token \ + when the prompt opens a `<|channel>thought` block; outcome={outcome:?}" + ); + } + if usage.reasoning_tokens == 0 { + bail!( + "Gemma 4 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "mmproj-F16.gguf"), +)] +fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const MAX_GENERATED_TOKENS: i32 = 512; + + let model = fixture.model; + let backend = fixture.backend; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .context("image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let marker = mtmd_default_marker()?; + let messages = [ + LlamaChatMessage::new( + "system".to_owned(), + "# HOW YOU SHOULD THINK AND ANSWER\n\n\ + First draft your thinking process (inner monologue) until you arrive at a response. \ + Format your response using Markdown, and use LaTeX for any mathematical equations. \ + Write both your thoughts and the response in the same language as the input.\n\n\ + Your thinking process must follow the template below:\ + [THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. \ + Be as casual and as long as you want until you are confident to generate the response \ + to the user.[/THINK]Here, provide a self-contained response." + .to_owned(), + )?, + LlamaChatMessage::new("user".to_owned(), format!("{marker}{PROMPT_QUESTION}"))?, + ]; + let session = ChatSession::new( + model, + model.chat_templates()?, + &messages, + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + let input_text = MtmdInputText { + text: session.prompt()?, + add_special: true, + parse_special: true, + }; + + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let n_past = classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut context, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + )?; + + let mut sampler = LlamaSampler::greedy()?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: MAX_GENERATED_TOKENS, + } + .run(); + + let usage = classifier.usage(); + + if outcome.observed_reasoning == 0 { + bail!( + "Mistral 3 multimodal + thinking: classifier must emit at least one Reasoning token \ + when the model opens a `[THINK]` block; outcome={outcome:?}" + ); + } + if usage.reasoning_tokens == 0 { + bail!( + "Mistral 3 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn qwen_classifier_emits_reasoning_for_multimodal_thinking_prompt( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + const MAX_GENERATED_TOKENS: i32 = 200; + + let model = fixture.model; + let backend = fixture.backend; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let mut context = LlamaContext::from_model( + model, + backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .context("image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let session = media_question_chat_session(model, PROMPT_QUESTION)?; + let input_text = MtmdInputText { + text: session.prompt()?, + add_special: true, + parse_special: true, + }; + + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let n_past = classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut context, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + )?; + + let mut sampler = LlamaSampler::chain([ + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, + LlamaSampler::top_k(40)?, + LlamaSampler::top_p(0.9, 1)?, + LlamaSampler::min_p(0.05, 1)?, + LlamaSampler::temp(0.7)?, + LlamaSampler::dist(0x00C0_FFEE)?, + ])?; + + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + let outcome = ClassifySampleLoop { + classifier: &mut classifier, + sampler: &mut sampler, + context: &mut context, + batch: &mut batch, + initial_position: n_past, + max_generated_tokens: MAX_GENERATED_TOKENS, + } + .run(); + + let usage = classifier.usage(); + + if outcome.observed_reasoning == 0 { + bail!( + "Qwen multimodal + thinking: classifier must emit at least one Reasoning token \ + when the prompt opens a `` block; outcome={outcome:?}" + ); + } + if usage.reasoning_tokens == 0 { + bail!( + "Qwen multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" + ); + } + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs new file mode 100644 index 000000000..1414447bc --- /dev/null +++ b/llama-cpp-bindings-tests/tests/multimodal_usage_accounting.rs @@ -0,0 +1,315 @@ +use anyhow::Context as _; +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::eval_multimodal_chunks_params::EvalMultimodalChunksParams; +use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunk::FittedMtmdInputChunk; +use llama_cpp_bindings::mtmd::fitted_mtmd_input_chunks::FittedMtmdInputChunks; +use llama_cpp_bindings::mtmd::mtmd_bitmap::MtmdBitmap; +use llama_cpp_bindings::mtmd::mtmd_default_marker::mtmd_default_marker; +use llama_cpp_bindings::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use llama_cpp_bindings::mtmd::mtmd_input_text::MtmdInputText; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; +use llama_cpp_bindings_tests::evaluated_multimodal_usage::EvaluatedMultimodalUsage; +use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; +use llama_cpp_bindings_tests::prompt_question::PROMPT_QUESTION; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +fn build_multimodal_chunks_and_eval_into_usage( + fixture: &LlamaFixture<'_>, +) -> Result { + let model = fixture.model; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .context("image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let marker = mtmd_default_marker()?; + let prompt = format!("{marker}{PROMPT_QUESTION}"); + + let input_text = MtmdInputText { + text: prompt, + add_special: false, + parse_special: true, + }; + + let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; + let expected = ChunkTokenBreakdown::from_chunks(&chunks); + + let context_params = (*fixture.context_params).into_llama_context_params(); + let mut context = LlamaContext::from_model(model, fixture.backend, context_params)?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + classifier.eval_multimodal_chunks( + &chunks, + mtmd_ctx, + &mut context, + EvalMultimodalChunksParams { + start_position: 0, + seq_id: 0, + n_batch: 512, + logits_last: true, + }, + )?; + + Ok(EvaluatedMultimodalUsage { + usage: classifier.into_usage(), + chunk_breakdown: expected, + }) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn prompt_tokens_match_text_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { + let EvaluatedMultimodalUsage { + usage, + chunk_breakdown: expected, + } = build_multimodal_chunks_and_eval_into_usage(fixture)?; + + if usage.prompt_tokens != expected.text { + bail!( + "prompt_tokens must equal sum of text-chunk n_tokens; expected {}, got {}", + expected.text, + usage.prompt_tokens + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn input_image_tokens_match_image_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { + let EvaluatedMultimodalUsage { + usage, + chunk_breakdown: expected, + } = build_multimodal_chunks_and_eval_into_usage(fixture)?; + + if usage.input_image_tokens != expected.image { + bail!( + "input_image_tokens must equal sum of image-chunk n_tokens; expected {}, got {}", + expected.image, + usage.input_image_tokens + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn input_audio_tokens_are_zero_for_image_only_input(fixture: &LlamaFixture<'_>) -> Result<()> { + let EvaluatedMultimodalUsage { + usage, + chunk_breakdown: expected, + } = build_multimodal_chunks_and_eval_into_usage(fixture)?; + + if expected.audio != 0 { + bail!( + "fixture invariant: image-only multimodal input should produce zero audio chunk tokens, got {}", + expected.audio + ); + } + if usage.input_audio_tokens != 0 { + bail!( + "input_audio_tokens must be zero when no audio chunks are evaluated; got {}", + usage.input_audio_tokens + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn completion_tokens_are_zero_after_eval_before_generation( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let EvaluatedMultimodalUsage { usage, .. } = + build_multimodal_chunks_and_eval_into_usage(fixture)?; + + if usage.completion_tokens() != 0 { + bail!( + "completion_tokens must be zero immediately after eval (no generation has occurred); got {}", + usage.completion_tokens() + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn text_chunk_records_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let input_text = MtmdInputText { + text: "hello world".to_owned(), + add_special: false, + parse_special: false, + }; + let llama_ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_chunks = FittedMtmdInputChunks::fit( + mtmd_ctx.tokenize(input_text, &[])?, + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + + let text_chunk = (0..fitted_chunks.len()) + .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) + .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Text) + .context("text-only tokenization should produce at least one text chunk")?; + + let n_tokens = u64::try_from(text_chunk.chunk().n_tokens())?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + + classifier.ingest_fitted_chunk(&text_chunk); + + let usage = classifier.usage(); + if usage.prompt_tokens != n_tokens { + bail!( + "text chunk must record n_tokens as prompt_tokens; expected {n_tokens}, got {}", + usage.prompt_tokens + ); + } + if usage.input_image_tokens != 0 { + bail!( + "text chunk must not bump input_image_tokens; got {}", + usage.input_image_tokens + ); + } + if usage.input_audio_tokens != 0 { + bail!( + "text chunk must not bump input_audio_tokens; got {}", + usage.input_audio_tokens + ); + } + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +fn image_chunk_records_input_image_tokens_only(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + + let image_path = fixtures_dir().join("llamas.jpg"); + let image_path_str = image_path + .to_str() + .context("image path is not valid UTF-8")?; + let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; + + let marker = mtmd_default_marker()?; + let input_text = MtmdInputText { + text: marker.to_owned(), + add_special: false, + parse_special: true, + }; + let llama_ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_chunks = FittedMtmdInputChunks::fit( + mtmd_ctx.tokenize(input_text, &[&bitmap])?, + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + + let image_chunk = (0..fitted_chunks.len()) + .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) + .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Image) + .context("multimodal tokenization should produce an image chunk")?; + + let n_tokens = u64::try_from(image_chunk.chunk().n_tokens())?; + if n_tokens == 0 { + bail!("image chunk should report at least one token"); + } + + let mut classifier = SampledTokenClassifier::undeterminable(model); + + classifier.ingest_fitted_chunk(&image_chunk); + + let usage = classifier.usage(); + if usage.input_image_tokens != n_tokens { + bail!( + "image chunk must record n_tokens as input_image_tokens; expected {n_tokens}, got {}", + usage.input_image_tokens + ); + } + if usage.prompt_tokens != 0 { + bail!( + "image chunk must not bump prompt_tokens; got {}", + usage.prompt_tokens + ); + } + if usage.input_audio_tokens != 0 { + bail!( + "image chunk must not bump input_audio_tokens; got {}", + usage.input_audio_tokens + ); + } + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/multimodal_vision.rs b/llama-cpp-bindings-tests/tests/multimodal_vision.rs deleted file mode 100644 index bc7662ecb..000000000 --- a/llama-cpp-bindings-tests/tests/multimodal_vision.rs +++ /dev/null @@ -1,2124 +0,0 @@ -use anyhow::Context; -use anyhow::Result; -use anyhow::anyhow; -use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::EvalMultimodalChunksParams; -use llama_cpp_bindings::TokenUsage; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::error::EvalMultimodalChunksError; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::mtmd::FittedMtmdInputChunk; -use llama_cpp_bindings::mtmd::FittedMtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdBitmap; -use llama_cpp_bindings::mtmd::MtmdContext; -use llama_cpp_bindings::mtmd::MtmdContextParams; -use llama_cpp_bindings::mtmd::MtmdEvalError; -use llama_cpp_bindings::mtmd::MtmdInputChunkType; -use llama_cpp_bindings::mtmd::MtmdInputChunks; -use llama_cpp_bindings::mtmd::MtmdInputText; -use llama_cpp_bindings::mtmd::NonCausalChunkMicroBatchMismatch; -use llama_cpp_bindings::mtmd::mtmd_default_marker; -use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_bindings::token::LlamaToken; -use llama_cpp_bindings_tests::build_user_prompt_with_media_marker::build_user_prompt_with_media_marker; -use llama_cpp_bindings_tests::chunk_token_breakdown::ChunkTokenBreakdown; -use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; -use llama_cpp_bindings_tests::fixtures_dir::fixtures_dir; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn from_buffer_creates_bitmap_from_image_bytes(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let fixtures = fixtures_dir(); - let image_path = fixtures.join("llamas.jpg"); - let image_bytes = std::fs::read(&image_path)?; - let bitmap = MtmdBitmap::from_buffer(mtmd_ctx, &image_bytes)?; - - assert!(bitmap.nx() > 0); - assert!(bitmap.ny() > 0); - assert!(!bitmap.is_audio()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let result = MtmdBitmap::from_file(mtmd_ctx, "path\0null"); - - assert!(result.is_err()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn copy_creates_owned_duplicate(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: "Hello <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - let copied = first_chunk.copy()?; - - assert!(copied.owned); - assert_eq!(copied.n_tokens(), first_chunk.n_tokens()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn encode_chunk_succeeds_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: "Describe: <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { - let result = mtmd_ctx.encode_chunk(&chunk); - assert!(result.is_ok()); - return Ok(()); - } - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn decode_use_non_causal_returns_bool_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: "Describe: <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { - let value = mtmd_ctx.decode_use_non_causal(&chunk); - let printed = format!("{value:?}"); - assert!( - !printed.is_empty(), - "decode_use_non_causal must return a Debug-printable bool" - ); - return Ok(()); - } - } - bail!("tokenization should produce at least one Image chunk"); -} - -fn tokenize_synthetic(fixture: &LlamaFixture<'_>, prompt: &str) -> Result { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: prompt.to_owned(), - add_special: true, - parse_special: true, - }; - Ok(mtmd_ctx.tokenize(input_text, &[&bitmap])?) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - let tokens = first_chunk - .text_tokens_of(first_chunk.chunk_type()?) - .ok_or_else(|| anyhow!("the first chunk of a text prompt must carry tokens"))?; - - assert_ne!(tokens, Vec::::new()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - assert!(first_chunk.n_tokens() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello world <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - assert!(first_chunk.n_positions() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; - let first_chunk = chunks - .get(0) - .ok_or_else(|| anyhow!("missing first chunk"))?; - assert_eq!(first_chunk.chunk_type()?, MtmdInputChunkType::Text); - assert!(first_chunk.id().is_none()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type()? == MtmdInputChunkType::Image { - assert_eq!(chunk.text_tokens_of(MtmdInputChunkType::Image), None); - - return Ok(()); - } - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn image_chunk_id_returns_some(fixture: &LlamaFixture<'_>) -> Result<()> { - let chunks = tokenize_synthetic(fixture, "Hello <__media__>")?; - for chunk_index in 0..chunks.len() { - let chunk = chunks - .get(chunk_index) - .ok_or_else(|| anyhow!("missing chunk at index {chunk_index}"))?; - if chunk.chunk_type() == Ok(MtmdInputChunkType::Image) { - assert!(chunk.id().is_some()); - return Ok(()); - } - } - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn init_and_supports_vision(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - assert!(mtmd_ctx.support_vision()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn init_from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_params = MtmdContextParams::default(); - let result = MtmdContext::init_from_file("path\0null", fixture.model, &mtmd_params); - - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn decode_use_mrope_is_true_for_qwen_vision(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - assert!( - mtmd_ctx.decode_use_mrope(), - "Qwen 3.5 / 3.6 mmproj uses mrope; decode_use_mrope must return true" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn support_audio_is_false_for_vision_only_mmproj(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - assert!( - !mtmd_ctx.support_audio(), - "Qwen 3.5 / 3.6 mmproj is vision-only; support_audio must return false" - ); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn get_audio_sample_rate_is_none_for_vision_only_mmproj(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - assert!( - mtmd_ctx.get_audio_sample_rate().is_none(), - "Qwen 3.5 / 3.6 mmproj has no audio; get_audio_sample_rate must return None" - ); - Ok(()) -} - -fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; (width as usize) * (height as usize) * 3]; - let bitmap = MtmdBitmap::from_image_data(width, height, &image_data)?; - let input_text = MtmdInputText { - text: "Describe: <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let n_positions = chunks.total_positions(); - let required_n_ctx = u32::try_from(n_positions + 256)?; - if fixture.context_params.n_ctx < required_n_ctx { - bail!( - "fixture n_ctx ({}) below required ({}) for {}x{} image", - fixture.context_params.n_ctx, - required_n_ctx, - width, - height, - ); - } - - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let n_batch = i32::try_from(llama_ctx.n_batch())?; - chunks.eval_chunks(mtmd_ctx, &llama_ctx, 0, 0, n_batch, false)?; - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 64, - n_batch = 64, - n_ubatch = 32, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 64, - n_batch = 64, - n_ubatch = 32, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let chunks = MtmdInputChunks::new()?; - let huge_batch = i32::try_from(llama_ctx.n_batch() + 1)?; - - let result = chunks.eval_chunks(mtmd_ctx, &llama_ctx, 0, 0, huge_batch, false); - - assert_eq!( - result, - Err(MtmdEvalError::BatchSizeExceedsContextLimit { - requested: huge_batch, - context_max: llama_ctx.n_batch(), - }) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!( - chunks.eval_chunks(mtmd_ctx, &llama_ctx, 0, 0, 0, true), - Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn fitting_no_chunks_leaves_nothing_to_evaluate(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = - FittedMtmdInputChunks::fit(MtmdInputChunks::new()?, mtmd_ctx, llama_ctx.n_ubatch())?; - - assert!(fitted_chunks.is_empty()); - assert!(FittedMtmdInputChunk::at(&fitted_chunks, 0).is_none()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = FittedMtmdInputChunks::fit( - tokenize_synthetic(fixture, "Describe: <__media__>")?, - mtmd_ctx, - llama_ctx.n_ubatch(), - )?; - let first_chunk = - FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; - - assert_eq!( - first_chunk.eval_single(mtmd_ctx, &llama_ctx, 0, 0, 0, true), - Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluating( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_micro_batch_tokens = llama_ctx.n_ubatch(); - let fitted_chunks = FittedMtmdInputChunks::fit( - tokenize_synthetic(fixture, "Describe: <__media__>")?, - mtmd_ctx, - fitted_micro_batch_tokens, - )?; - let first_chunk = - FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; - let smaller_micro_batch_tokens = fitted_micro_batch_tokens - 1; - - assert_eq!( - first_chunk.eval_single( - mtmd_ctx, - &llama_ctx, - 0, - 0, - i32::try_from(smaller_micro_batch_tokens)?, - true - ), - Err(MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted { - evaluated_micro_batch_tokens: smaller_micro_batch_tokens, - fitted_micro_batch_tokens, - }) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn classifier_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let mut classifier = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Ignore)?; - - assert!(matches!( - classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &llama_ctx, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 0, - logits_last: true, - }, - ), - Err(EvalMultimodalChunksError::EvalFailed( - MtmdEvalError::NonPositiveBatchSize { requested: 0 } - )) - )); - assert_eq!(classifier.usage().prompt_tokens, 0); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_with_standard_image(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let fixtures = fixtures_dir(); - let image_path = fixtures.join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - let input_text = MtmdInputText { - text: "What is in this image? <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let n_positions = chunks.total_positions(); - let required_n_ctx = u32::try_from(n_positions + 256)?; - assert!( - fixture.context_params.n_ctx >= required_n_ctx, - "fixture n_ctx ({}) below required ({}); update the attribute literal", - fixture.context_params.n_ctx, - required_n_ctx, - ); - - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let n_batch = i32::try_from(llama_ctx.n_batch())?; - let result = chunks.eval_chunks(mtmd_ctx, &llama_ctx, 0, 0, n_batch, false); - - assert!(result.is_ok()); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_with_varied_dimensions(fixture: &LlamaFixture<'_>) -> Result<()> { - let test_dimensions: [(u32, u32); 4] = [(224, 224), (512, 512), (100, 500), (337, 421)]; - - for (width, height) in test_dimensions { - let result = eval_synthetic_bitmap(fixture, width, height); - assert!( - result.is_ok(), - "dimension {width}x{height} should succeed: {result:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn eval_chunks_supports_extreme_dimensions(fixture: &LlamaFixture<'_>) -> Result<()> { - let extreme_dimensions: [(u32, u32); 6] = [ - (2, 2), - (7, 13), - (3, 1000), - (1000, 3), - (1920, 1080), - (4096, 4096), - ]; - - for (width, height) in extreme_dimensions { - eval_synthetic_bitmap(fixture, width, height) - .with_context(|| format!("dimension {width}x{height} should succeed"))?; - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn tokenize_text_with_image(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let input_text = MtmdInputText { - text: "Describe this image: <__media__>".to_string(), - add_special: true, - parse_special: true, - }; - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - assert!(!chunks.is_empty()); - assert!(chunks.total_tokens() > 0); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn tokenize_bitmap_count_mismatch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let input_text = MtmdInputText { - text: "No media markers here".to_string(), - add_special: true, - parse_special: true, - }; - let image_data = vec![128u8; 64 * 64 * 3]; - let bitmap = MtmdBitmap::from_image_data(64, 64, &image_data)?; - let result = mtmd_ctx.tokenize(input_text, &[&bitmap]); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn tokenize_with_null_byte_in_text_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let input_text = MtmdInputText { - text: "text\0null".to_string(), - add_special: true, - parse_special: true, - }; - let result = mtmd_ctx.tokenize(input_text, &[]); - assert!(result.is_err()); - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn multimodal_vision_inference_produces_output(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - ) - .with_context(|| "unable to create llama context")?; - - assert!( - mtmd_ctx.support_vision(), - "model should support vision input" - ); - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .with_context(|| "image path is not valid UTF-8")?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str) - .with_context(|| "failed to load image from file")?; - - let formatted_prompt = - build_user_prompt_with_media_marker(model, "What animals do you see in this image?")?; - - let input_text = MtmdInputText { - text: formatted_prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx - .tokenize(input_text, &[&bitmap]) - .with_context(|| "failed to tokenize multimodal input")?; - - assert!( - !chunks.is_empty(), - "tokenization should produce at least one chunk" - ); - - let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; - - eprintln!( - "tokenized into {} chunks, text {} image {} audio {}", - chunks.len(), - expected.text, - expected.image, - expected.audio - ); - - assert!( - expected.image > 0, - "vision input must produce at least one image chunk" - ); - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier - .eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &ctx, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - ) - .with_context(|| "failed to evaluate chunks")?; - - eprintln!("evaluated chunks, n_past = {n_past}"); - - { - let usage = classifier.usage(); - assert_eq!(usage.prompt_tokens, expected.text); - assert_eq!(usage.input_image_tokens, expected.image); - assert_eq!(usage.input_audio_tokens, expected.audio); - } - - let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - let totals = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut ctx, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: 512, - } - .run()?; - - eprintln!("generated text: {}", totals.generated_raw); - - assert!( - !totals.generated_raw.is_empty(), - "model should generate at least one token from image input" - ); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, expected.text); - assert_eq!(usage.input_image_tokens, expected.image); - assert_eq!(usage.input_audio_tokens, expected.audio); - assert_eq!(usage.content_tokens, totals.observed_content); - assert_eq!(usage.reasoning_tokens, totals.observed_reasoning); - assert_eq!( - usage.completion_tokens(), - totals.observed_content + totals.observed_reasoning - ); - - Ok(()) -} - -const PROMPT_QUESTION: &str = "What animals do you see in this image?"; - -fn build_multimodal_chunks_and_eval_into_usage( - fixture: &LlamaFixture<'_>, -) -> Result<(TokenUsage, ChunkTokenBreakdown)> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!("{marker}{PROMPT_QUESTION}"); - - let input_text = MtmdInputText { - text: prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - let expected = ChunkTokenBreakdown::from_chunks(&chunks)?; - - let context_params = (*fixture.context_params).into_llama_context_params(); - let context = LlamaContext::from_model(model, fixture.backend, context_params)?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - Ok((classifier.into_usage(), expected)) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn prompt_tokens_match_text_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { - let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; - - if usage.prompt_tokens != expected.text { - bail!( - "prompt_tokens must equal sum of text-chunk n_tokens; expected {}, got {}", - expected.text, - usage.prompt_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn input_image_tokens_match_image_chunk_total(fixture: &LlamaFixture<'_>) -> Result<()> { - let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; - - if usage.input_image_tokens != expected.image { - bail!( - "input_image_tokens must equal sum of image-chunk n_tokens; expected {}, got {}", - expected.image, - usage.input_image_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn input_audio_tokens_are_zero_for_image_only_input(fixture: &LlamaFixture<'_>) -> Result<()> { - let (usage, expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; - - if expected.audio != 0 { - bail!( - "fixture invariant: image-only multimodal input should produce zero audio chunk tokens, got {}", - expected.audio - ); - } - if usage.input_audio_tokens != 0 { - bail!( - "input_audio_tokens must be zero when no audio chunks are evaluated; got {}", - usage.input_audio_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn completion_tokens_are_zero_after_eval_before_generation( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let (usage, _expected) = build_multimodal_chunks_and_eval_into_usage(fixture)?; - - if usage.completion_tokens() != 0 { - bail!( - "completion_tokens must be zero immediately after eval (no generation has occurred); got {}", - usage.completion_tokens() - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_records_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let input_text = MtmdInputText { - text: "hello world".to_owned(), - add_special: false, - parse_special: false, - }; - let llama_ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = FittedMtmdInputChunks::fit( - mtmd_ctx.tokenize(input_text, &[])?, - mtmd_ctx, - llama_ctx.n_ubatch(), - )?; - - let text_chunk = (0..fitted_chunks.len()) - .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) - .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Text) - .ok_or_else(|| anyhow!("text-only tokenization should produce at least one text chunk"))?; - - let n_tokens = u64::try_from(text_chunk.chunk().n_tokens())?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - - classifier.ingest_fitted_chunk(&text_chunk); - - let usage = classifier.usage(); - if usage.prompt_tokens != n_tokens { - bail!( - "text chunk must record n_tokens as prompt_tokens; expected {n_tokens}, got {}", - usage.prompt_tokens - ); - } - if usage.input_image_tokens != 0 { - bail!( - "text chunk must not bump input_image_tokens; got {}", - usage.input_image_tokens - ); - } - if usage.input_audio_tokens != 0 { - bail!( - "text chunk must not bump input_audio_tokens; got {}", - usage.input_audio_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn image_chunk_records_input_image_tokens_only(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let input_text = MtmdInputText { - text: marker.to_owned(), - add_special: false, - parse_special: true, - }; - let llama_ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = FittedMtmdInputChunks::fit( - mtmd_ctx.tokenize(input_text, &[&bitmap])?, - mtmd_ctx, - llama_ctx.n_ubatch(), - )?; - - let image_chunk = (0..fitted_chunks.len()) - .filter_map(|index| FittedMtmdInputChunk::at(&fitted_chunks, index)) - .find(|chunk| chunk.chunk_type() == MtmdInputChunkType::Image) - .ok_or_else(|| anyhow!("multimodal tokenization should produce an image chunk"))?; - - let n_tokens = u64::try_from(image_chunk.chunk().n_tokens())?; - if n_tokens == 0 { - bail!("image chunk should report at least one token"); - } - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - - classifier.ingest_fitted_chunk(&image_chunk); - - let usage = classifier.usage(); - if usage.input_image_tokens != n_tokens { - bail!( - "image chunk must record n_tokens as input_image_tokens; expected {n_tokens}, got {}", - usage.input_image_tokens - ); - } - if usage.prompt_tokens != 0 { - bail!( - "image chunk must not bump prompt_tokens; got {}", - usage.prompt_tokens - ); - } - if usage.input_audio_tokens != 0 { - bail!( - "image chunk must not bump input_audio_tokens; got {}", - usage.input_audio_tokens - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 2048, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -fn text_chunk_drives_marker_state_machine_to_reasoning(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let input_text = MtmdInputText { - text: "<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n\n".to_owned(), - add_special: false, - parse_special: true, - }; - let llama_ctx = LlamaContext::from_model( - model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let fitted_chunks = FittedMtmdInputChunks::fit( - mtmd_ctx.tokenize(input_text, &[])?, - mtmd_ctx, - llama_ctx.n_ubatch(), - )?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - - for index in 0..fitted_chunks.len() { - let chunk = FittedMtmdInputChunk::at(&fitted_chunks, index) - .ok_or_else(|| anyhow!("chunk index {index} must exist"))?; - classifier.ingest_fitted_chunk(&chunk); - } - - if classifier.current_section() != llama_cpp_bindings::SampledTokenSection::Reasoning { - bail!( - "text chunk replay must transition the classifier section to Reasoning when the \ - prompt opens a `` block; got {:?}", - classifier.current_section() - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "mmproj-F16.gguf"), -)] -fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - let model = fixture.model; - let backend = fixture.backend; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!( - "user\n{marker}What animals do you see in this image?\nmodel\n<|channel>thought\n" - ); - - let input_text = MtmdInputText { - text: prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - if outcome.observed_reasoning == 0 { - bail!( - "Gemma 4 multimodal + thinking: classifier must emit at least one Reasoning token \ - when the prompt opens a `<|channel>thought` block; outcome={outcome:?}" - ); - } - if usage.reasoning_tokens == 0 { - bail!( - "Gemma 4 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "mmproj-F16.gguf"), -)] -fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - let model = fixture.model; - let backend = fixture.backend; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!( - "[SYSTEM_PROMPT]# HOW YOU SHOULD THINK AND ANSWER\n\n\ - First draft your thinking process (inner monologue) until you arrive at a response. \ - Format your response using Markdown, and use LaTeX for any mathematical equations. \ - Write both your thoughts and the response in the same language as the input.\n\n\ - Your thinking process must follow the template below:\ - [THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. \ - Be as casual and as long as you want until you are confident to generate the response \ - to the user.[/THINK]Here, provide a self-contained response.[/SYSTEM_PROMPT]\ - [INST]{marker}What animals do you see in this image?[/INST]" - ); - - let input_text = MtmdInputText { - text: prompt, - add_special: true, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - let mut sampler = LlamaSampler::greedy()?; - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - if outcome.observed_reasoning == 0 { - bail!( - "Mistral 3 multimodal + thinking: classifier must emit at least one Reasoning token \ - when the model opens a `[THINK]` block; outcome={outcome:?}" - ); - } - if usage.reasoning_tokens == 0 { - bail!( - "Mistral 3 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), -)] -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 4096, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - let model = fixture.model; - let backend = fixture.backend; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!( - "<|im_start|>user\n{marker}What animals do you see in this image?<|im_end|>\n<|im_start|>assistant\n\n" - ); - - let input_text = MtmdInputText { - text: prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - if outcome.observed_reasoning == 0 { - bail!( - "Qwen 3.5 multimodal + thinking: classifier must emit at least one Reasoning token \ - when the prompt opens a `` block; outcome={outcome:?}" - ); - } - if usage.reasoning_tokens == 0 { - bail!( - "Qwen 3.5 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 512, - n_ubatch = 512, - mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), -)] -fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - let model = fixture.model; - let backend = fixture.backend; - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .ok_or_else(|| anyhow!("image path is not valid UTF-8"))?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - let marker = mtmd_default_marker()?; - let prompt = format!( - "<|im_start|>user\n{marker}What animals do you see in this image?<|im_end|>\n<|im_start|>assistant\n\n" - ); - - let input_text = MtmdInputText { - text: prompt, - add_special: false, - parse_special: true, - }; - - let chunks = mtmd_ctx.tokenize(input_text, &[&bitmap])?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let n_past = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &context, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: 512, - logits_last: true, - }, - )?; - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position: n_past, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - if outcome.observed_reasoning == 0 { - bail!( - "Qwen 3.6 multimodal + thinking: classifier must emit at least one Reasoning token; outcome={outcome:?}" - ); - } - if usage.reasoning_tokens == 0 { - bail!( - "Qwen 3.6 multimodal + thinking: usage.reasoning_tokens must be non-zero; usage={usage:?}" - ); - } - - Ok(()) -} - -fn tokenize_question_about_llamas( - fixture: &LlamaFixture<'_>, - mtmd_ctx: &MtmdContext, -) -> Result { - let image_path = fixtures_dir().join("llamas.jpg"); - let image_path_str = image_path - .to_str() - .with_context(|| "image path is not valid UTF-8")?; - let bitmap = MtmdBitmap::from_file(mtmd_ctx, image_path_str)?; - - Ok(mtmd_ctx.tokenize( - MtmdInputText { - text: build_user_prompt_with_media_marker(fixture.model, PROMPT_QUESTION)?, - add_special: false, - parse_special: true, - }, - &[&bitmap], - )?) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 2048, - n_ubatch = 128, - mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), -)] -fn gemma3_eval_chunks_rejects_a_non_causal_image_larger_than_the_micro_batch_before_evaluating( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; - let image_tokens = ChunkTokenBreakdown::from_chunks(&chunks)?.image; - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!( - chunks.eval_chunks( - mtmd_ctx, - &llama_ctx, - 0, - 0, - i32::try_from(llama_ctx.n_batch())?, - true - ), - Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( - NonCausalChunkMicroBatchMismatch { - chunk_tokens: usize::try_from(image_tokens)?, - micro_batch_tokens: llama_ctx.n_ubatch(), - } - )) - ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 2048, - n_ubatch = 128, - mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), -)] -fn gemma3_fitting_rejects_a_non_causal_image_larger_than_the_micro_batch( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; - let image_chunk = (0..chunks.len()) - .filter_map(|chunk_index| chunks.get(chunk_index)) - .find(|chunk| chunk.chunk_type() == Ok(MtmdInputChunkType::Image)) - .context("the prompt must contain an image chunk")?; - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - assert_eq!( - image_chunk.fit_to_micro_batch(mtmd_ctx, llama_ctx.n_ubatch()), - Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( - NonCausalChunkMicroBatchMismatch { - chunk_tokens: image_chunk.n_tokens(), - micro_batch_tokens: llama_ctx.n_ubatch(), - } - )) - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "gemma-3-4b-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 2048, - n_ubatch = 128, - mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), -)] -fn gemma3_classifier_leaves_the_prompt_untouched_when_a_non_causal_image_exceeds_the_micro_batch( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let mtmd_ctx = fixture - .mtmd_context - .expect("mmproj_file declared in attribute"); - let chunks = tokenize_question_about_llamas(fixture, mtmd_ctx)?; - let llama_ctx = LlamaContext::from_model( - fixture.model, - fixture.backend, - (*fixture.context_params).into_llama_context_params(), - )?; - let mut classifier = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Ignore)?; - - let evaluation = classifier.eval_multimodal_chunks( - &chunks, - mtmd_ctx, - &llama_ctx, - EvalMultimodalChunksParams { - start_position: 0, - seq_id: 0, - n_batch: i32::try_from(llama_ctx.n_batch())?, - logits_last: true, - }, - ); - - assert!(matches!( - evaluation, - Err(EvalMultimodalChunksError::EvalFailed( - MtmdEvalError::NonCausalChunkExceedsMicroBatch(NonCausalChunkMicroBatchMismatch { - micro_batch_tokens, - .. - }) - )) if micro_batch_tokens == llama_ctx.n_ubatch() - )); - assert_eq!(classifier.usage().prompt_tokens, 0); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/parameter_fitting.rs b/llama-cpp-bindings-tests/tests/parameter_fitting.rs new file mode 100644 index 000000000..deac94da7 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/parameter_fitting.rs @@ -0,0 +1,163 @@ +use std::ffi::CString; +use std::num::NonZeroU32; +use std::pin::pin; + +use anyhow::Result; + +use llama_cpp_bindings::context::params::LlamaContextParams; +use llama_cpp_bindings::max_devices::max_devices; +use llama_cpp_bindings::model::params::LlamaModelParams; +use llama_cpp_bindings::model::params::fit_extra_model::FitExtraModel; +use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; +use llama_cpp_bindings::mtmd::mtmd_memory_usage::MtmdMemoryUsage; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; +use llama_cpp_test_model_provisioning::gguf_source::GgufSource; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn fit_params_fits_a_nonzero_context_for_the_model(fixture: &LlamaFixture<'_>) -> Result<()> { + let model_path_c = CString::new(fixture.model_path.as_os_str().as_encoded_bytes())?; + + let mut params = pin!(LlamaModelParams::default()); + let mut context_params = LlamaContextParams::default(); + let mut margins = vec![0usize; max_devices()]; + + let result = params.as_mut().fit_params( + &model_path_c, + &mut context_params, + &mut margins, + 512, + None, + llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, + ); + + let fit = result?; + assert!(fit.n_ctx > 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn fit_params_aligns_the_extra_model_context_with_the_fitted_one( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model_path_c = CString::new(fixture.model_path.as_os_str().as_encoded_bytes())?; + + let mut params = pin!(LlamaModelParams::default()); + let mut context_params = LlamaContextParams::default(); + let mut margins = vec![0usize; max_devices()]; + + let unfittable_extra_n_ctx = NonZeroU32::MIN; + let mut extra_params = pin!(LlamaModelParams::default()); + let mut extra_context_params = + LlamaContextParams::default().with_n_ctx(Some(unfittable_extra_n_ctx)); + let mut extra = FitExtraModel { + model_path: &model_path_c, + model_params: extra_params.as_mut(), + context_params: &mut extra_context_params, + shares_model: true, + }; + + let result = params.as_mut().fit_params( + &model_path_c, + &mut context_params, + &mut margins, + 512, + Some(&mut extra), + llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, + ); + + let fit = result?; + assert!(fit.n_ctx > 0); + assert_ne!( + extra_context_params.n_ctx(), + Some(unfittable_extra_n_ctx), + "the llama.cpp fit must overwrite the extra model's context size" + ); + assert_eq!( + extra_context_params.context_params.n_ctx, + context_params.context_params.n_ctx + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn fit_params_fits_the_model_beside_the_projector_memory_reserved_on_its_device( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mmproj_path = GgufSource::HuggingFace { + repo: "unsloth/Qwen3.5-0.8B-GGUF", + file: "mmproj-F16.gguf", + } + .resolve_path()?; + let model_path_c = CString::new(fixture.model_path.as_os_str().as_encoded_bytes())?; + let mmproj_path_c = CString::new(mmproj_path.as_os_str().as_encoded_bytes())?; + + let mut params = pin!(LlamaModelParams::default()); + let mut context_params = LlamaContextParams::default(); + let mut margins = vec![0usize; max_devices()]; + + MtmdMemoryUsage::measure(&mmproj_path_c, &MtmdContextParams::default())? + .add_to_fit_margins(&mut margins); + + assert_ne!(margins[0], 0, "{margins:?}"); + + let fit = params.as_mut().fit_params( + &model_path_c, + &mut context_params, + &mut margins, + 512, + None, + llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, + )?; + + assert!(fit.n_ctx > 0); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs new file mode 100644 index 000000000..d65830f1b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/prompt_token_accounting.rs @@ -0,0 +1,328 @@ +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::batch_add_error::BatchAddError; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings_types::token_usage_error::TokenUsageError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); + + classifier.feed_prompt_to_batch( + &mut batch, + model.token_bos().context("model declares a BOS token")?, + 0, + &[0], + false, + )?; + classifier.feed_prompt_to_batch( + &mut batch, + model.token_eos().context("model declares an EOS token")?, + 1, + &[0], + false, + )?; + + assert_eq!(classifier.pending_prompt_tokens(), 2); + assert_eq!(batch.n_tokens(), 2); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); + + let tokens = vec![ + model.token_bos().context("model declares a BOS token")?, + model.token_eos().context("model declares an EOS token")?, + model.token_nl().context("model declares a newline token")?, + ]; + classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; + + assert_eq!(classifier.pending_prompt_tokens(), 3); + assert_eq!(batch.n_tokens(), 3); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); + + classifier.feed_prompt_to_batch( + &mut batch, + model.token_bos().context("model declares a BOS token")?, + 0, + &[0], + false, + )?; + classifier.feed_prompt_to_batch( + &mut batch, + model.token_eos().context("model declares an EOS token")?, + 1, + &[0], + false, + )?; + + let promoted = classifier.commit_prompt_tokens(); + + assert_eq!(promoted, 2); + assert_eq!(classifier.pending_prompt_tokens(), 0); + assert_eq!(classifier.usage().prompt_tokens, 2); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn discard_pending_prompt_tokens_clears_count_without_recording_usage( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 8, + n_ubatch: 8, + n_seq_max: 1, + n_vocab: model.n_vocab(), + }); + + classifier.feed_prompt_to_batch( + &mut batch, + model.token_bos().context("model declares a BOS token")?, + 0, + &[0], + false, + )?; + + let discarded = classifier.discard_pending_prompt_tokens(); + + assert_eq!(discarded, 1); + assert_eq!(classifier.pending_prompt_tokens(), 0); + assert_eq!(classifier.usage().prompt_tokens, 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn cached_prompt_tokens_cannot_exceed_the_recorded_prompt(fixture: &LlamaFixture<'_>) { + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + + classifier.record_prompt_tokens(2); + + assert_eq!( + classifier.record_cached_prompt_tokens(3), + Err(TokenUsageError::CachedExceedsPrompt { + cached_after: 3, + prompt: 2, + }) + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_prompt_token_refused_by_a_full_batch_is_not_counted(fixture: &LlamaFixture<'_>) { + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 0, + n_ubatch: 0, + n_seq_max: 1, + n_vocab: fixture.model.n_vocab(), + }); + + assert_eq!( + classifier.feed_prompt_to_batch(&mut batch, LlamaToken::new(0), 0, &[0], false), + Err(BatchAddError::BatchFull { n_batch: 0 }) + ); + assert_eq!(classifier.pending_prompt_tokens(), 0); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn a_prompt_sequence_refused_by_a_full_batch_is_not_counted(fixture: &LlamaFixture<'_>) { + let mut classifier = SampledTokenClassifier::undeterminable(fixture.model); + let mut batch = LlamaBatch::new(ContextLimits { + n_batch: 0, + n_ubatch: 0, + n_seq_max: 1, + n_vocab: fixture.model.n_vocab(), + }); + + assert_eq!( + classifier.feed_prompt_sequence_to_batch(&mut batch, &[LlamaToken::new(0)], 0, false), + Err(BatchAddError::BatchFull { n_batch: 0 }) + ); + assert_eq!(classifier.pending_prompt_tokens(), 0); +} diff --git a/llama-cpp-bindings-tests/tests/reasoning_classification.rs b/llama-cpp-bindings-tests/tests/reasoning_classification.rs new file mode 100644 index 000000000..0c4723785 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/reasoning_classification.rs @@ -0,0 +1,334 @@ +use anyhow::Result; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; +use llama_cpp_bindings_tests::closed_reasoning_chat_session::closed_reasoning_chat_session; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const ARITHMETIC_QUESTION: &str = "What is 2 + 2?"; +const SINGLE_WORD_INSTRUCTION: &str = "Reply with the single word: four. Do not explain."; +const MAX_ANSWER_TOKENS: i32 = 200; +const MAX_REASONING_ANSWER_TOKENS: i32 = 512; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn deepseek_r1_8b_answers_without_reasoning_after_its_reasoning_is_closed( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = closed_reasoning_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]"); + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run(); + + answer.assert_content_without_reasoning(&session); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn deepseek_r1_8b_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run(); + + answer.assert_reasoning_matches_parse(&session, "\n"); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn gemma4_answers_without_reasoning_when_thinking_is_disabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]", false)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run(); + + answer.assert_content_without_reasoning(&session); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn gemma4_reasoning_streams_match_the_parsed_answer_when_reasoning_is_prefilled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = ChatSession::new( + fixture.model, + fixture.model.chat_templates()?, + &[LlamaChatMessage::new( + "user".to_owned(), + SINGLE_WORD_INSTRUCTION.to_owned(), + )?], + "[]", + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::ContinueReasoning( + "The user wants one word.".to_owned(), + ), + enable_thinking: true, + parallel_tool_calls: false, + tool_choice: ChatToolChoice::Auto, + }, + )?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run(); + + answer.assert_reasoning_matches_parse(&session, "The user wants one word."); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn glm47_answers_without_reasoning_when_thinking_is_disabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", false)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run(); + + answer.assert_content_without_reasoning(&session); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn glm47_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run(); + + answer.assert_reasoning_matches_parse(&session, ""); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn mistral3_answers_without_reasoning_after_its_reasoning_is_closed( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = closed_reasoning_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]"); + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run(); + + answer.assert_content_without_reasoning(&session); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn mistral3_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, SINGLE_WORD_INSTRUCTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run(); + + answer.assert_reasoning_matches_parse(&session, ""); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn qwen35_answers_without_reasoning_when_thinking_is_disabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", false)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run(); + + answer.assert_content_without_reasoning(&session); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn qwen35_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run(); + + answer.assert_reasoning_matches_parse(&session, ""); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn qwen36_answers_without_reasoning_when_thinking_is_disabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", false)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_ANSWER_TOKENS, + } + .run(); + + answer.assert_content_without_reasoning(&session); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 8192, + n_batch = 2048, + n_ubatch = 512, +)] +fn qwen36_reasoning_streams_match_the_parsed_answer_when_thinking_is_enabled( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, ARITHMETIC_QUESTION, "[]", true)?; + let answer = ChatAnswerGeneration { + fixture, + session: &session, + sampler: &mut LlamaSampler::greedy()?, + max_generated_tokens: MAX_REASONING_ANSWER_TOKENS, + } + .run(); + + answer.assert_reasoning_matches_parse(&session, ""); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/reranking.rs b/llama-cpp-bindings-tests/tests/reranking.rs new file mode 100644 index 000000000..730b025b3 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/reranking.rs @@ -0,0 +1,67 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings_tests::normalize::normalize; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const QUERY: &str = "What is machine learning?"; +const RELEVANT_DOCUMENT: &str = "Machine learning is a subset of artificial intelligence."; +const UNRELATED_DOCUMENT: &str = "The weather today is sunny and warm."; + +fn cosine_similarity(left_embedding: &[f32], right_embedding: &[f32]) -> f32 { + normalize(left_embedding) + .iter() + .zip(normalize(right_embedding).iter()) + .map(|(left, right)| left * right) + .sum::() +} + +#[llama_test( + model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + n_seq_max = 3, + n_threads_batch = 8, + embeddings = true, +)] +fn the_relevant_document_embeds_closer_to_the_query_than_an_unrelated_one( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + for (sequence_id, text) in (0..).zip([QUERY, RELEVANT_DOCUMENT, UNRELATED_DOCUMENT]) { + let tokens = + fixture + .model + .str_to_token(text, AddBos::Always, ParseSpecialTokens::Always)?; + + batch.add_sequence(&tokens, sequence_id, false)?; + } + + context.decode(&batch)?; + + let query_embedding = context.embeddings_seq_ith(0)?; + let relevant_similarity = cosine_similarity(query_embedding, context.embeddings_seq_ith(1)?); + let unrelated_similarity = cosine_similarity(query_embedding, context.embeddings_seq_ith(2)?); + + assert!( + relevant_similarity > unrelated_similarity, + "the relevant document must rank above the unrelated one; \ + relevant={relevant_similarity} unrelated={unrelated_similarity}" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs new file mode 100644 index 000000000..e1dec307f --- /dev/null +++ b/llama-cpp-bindings-tests/tests/sampled_token_ingestion.rs @@ -0,0 +1,346 @@ +use anyhow::Context as _; +use anyhow::Result; +use anyhow::bail; + +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::error::ingest_error::IngestError; +use llama_cpp_bindings::error::parse_chat_message_error::ParseChatMessageError; +use llama_cpp_bindings::error::sample_error::SampleError; +use llama_cpp_bindings::error::token_outside_vocabulary::TokenOutsideVocabulary; +use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; +use llama_cpp_bindings::generation_progress::GenerationProgress; +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampled_token_section::SampledTokenSection; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings::token_piece::TokenPiece; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_fault_injection::armed_fault::ArmedFault; +use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_session_classifier_starts_in_the_pending_section(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "hello", "[]", false)?; + let classifier = SampledTokenClassifier::for_chat_session(&session); + + assert_eq!(classifier.current_section(), SampledTokenSection::Pending); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn undeterminable_classifier_releases_each_token_with_its_visible_and_raw_piece( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let [ordinary_token] = + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] + else { + bail!("\"hello\" must be a single token"); + }; + + let mut outcomes = Vec::new(); + classifier.ingest(ordinary_token, &mut outcomes)?; + classifier.finish(&mut outcomes)?; + + assert_eq!( + outcomes, + [IngestOutcome { + sampled_token: SampledToken::Undeterminable(ordinary_token), + piece: TokenPiece::Visible("hello".to_owned()), + }] + ); + assert_eq!(classifier.usage().undeterminable_tokens, 1); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn finishing_a_chat_answer_settles_its_pending_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let session = user_turn_chat_session(model, "hello", "[]", false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let [ordinary_token] = + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] + else { + bail!("\"hello\" must be a single token"); + }; + + let mut outcomes = Vec::new(); + classifier.ingest(ordinary_token, &mut outcomes)?; + classifier.finish(&mut outcomes)?; + + assert_eq!( + outcomes, + [IngestOutcome { + sampled_token: SampledToken::Content(ordinary_token), + piece: TokenPiece::Visible("hello".to_owned()), + }] + ); + assert_eq!(classifier.usage().content_tokens, 1); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn ingest_counts_every_token_through_the_end_of_generation( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut classifier = SampledTokenClassifier::undeterminable(model); + let [ordinary_token] = + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?[..] + else { + bail!("\"hello\" must be a single token"); + }; + let mut outcomes = Vec::new(); + + classifier.ingest(ordinary_token, &mut outcomes)?; + + assert_eq!( + classifier.ingest( + model.token_eos().context("model declares an EOS token")?, + &mut outcomes + )?, + GenerationProgress::Ended + ); + + assert_eq!(classifier.usage().undeterminable_tokens, 2); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn ingesting_a_token_outside_the_vocabulary_reports_it_in_every_classifier( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let token = LlamaToken::new(model.n_vocab()); + let session = user_turn_chat_session(model, "hello", "[]", false)?; + let mut outcomes = Vec::new(); + let expected = Err(IngestError::Detokenize( + TokenToStringError::TokenOutsideVocabulary(TokenOutsideVocabulary { + token, + n_vocab: model.n_vocab(), + }), + )); + + assert_eq!( + SampledTokenClassifier::undeterminable(model).ingest(token, &mut outcomes), + expected + ); + assert_eq!( + SampledTokenClassifier::for_chat_session(&session).ingest(token, &mut outcomes), + expected + ); + assert_eq!(outcomes, []); + + Ok(()) +} + +fn failed_classification() -> IngestError { + IngestError::Classify(ParseChatMessageError::Reported { + message: injected_fault_message(), + }) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn ingesting_a_generated_token_reports_a_classification_llama_cpp_fails( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let generated = fixture + .model + .str_to_token("Hi", AddBos::Never, ParseSpecialTokens::Never)?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + + assert_eq!( + classifier.ingest(generated[0], &mut Vec::new()), + Err(failed_classification()) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn ending_the_generation_reports_a_classification_llama_cpp_fails( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let end_of_sequence = fixture + .model + .token_eos() + .context("the model declares an EOS token")?; + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + + assert_eq!( + classifier.ingest(end_of_sequence, &mut Vec::new()), + Err(failed_classification()) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn sampling_reports_a_classification_llama_cpp_fails(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let mut context = fixture.build_context()?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + classifier.feed_prompt_sequence_to_batch(&mut batch, &session.tokenize_prompt()?, 0, false)?; + context.decode(&batch)?; + + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_CHAT_SESSION_CLASSIFY_THROWS); + + assert_eq!( + classifier + .sample( + &mut LlamaSampler::greedy()?, + &context, + batch.n_tokens() - 1, + &mut Vec::new(), + ) + .err(), + Some(SampleError::Ingest(failed_classification())) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs new file mode 100644 index 000000000..49c5c297c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/sampler_token_acceptance.rs @@ -0,0 +1,53 @@ +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn a_greedy_chain_accepts_many_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; + let tokens = vec![ + fixture + .model + .token_bos() + .context("model declares a BOS token")?, + fixture + .model + .token_eos() + .context("model declares an EOS token")?, + ]; + + sampler.accept_many(&tokens)?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn a_greedy_chain_accepts_the_bos_token(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut sampler = LlamaSampler::chain([LlamaSampler::greedy()?])?; + + sampler.accept( + fixture + .model + .token_bos() + .context("model declares a BOS token")?, + )?; + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/session_file_path_validation.rs b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs new file mode 100644 index 000000000..5a3e72bb3 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/session_file_path_validation.rs @@ -0,0 +1,389 @@ +use std::ffi::CString; +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +use anyhow::Result; + +use llama_cpp_bindings::context::load_seq_state_error::LoadSeqStateError; +use llama_cpp_bindings::context::load_session_error::LoadSessionError; +use llama_cpp_bindings::context::save_seq_state_error::SaveSeqStateError; +use llama_cpp_bindings::context::save_session_error::SaveSessionError; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + + assert_eq!( + context.state_save_file(non_utf8_path, &[]).err(), + Some(SaveSessionError::PathToStrError( + non_utf8_path.to_path_buf() + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + + assert_eq!( + context.state_load_file(non_utf8_path, 512).err(), + Some(LoadSessionError::PathToStrError( + non_utf8_path.to_path_buf() + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_save_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + + assert_eq!( + context.state_seq_save_file(non_utf8_path, 0, &[]).err(), + Some(SaveSeqStateError::PathToStrError( + non_utf8_path.to_path_buf() + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_load_file_refuses_a_path_that_is_not_utf8(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.bin")); + + assert_eq!( + context.state_seq_load_file(non_utf8_path, 0, 512).err(), + Some(LoadSeqStateError::PathToStrError( + non_utf8_path.to_path_buf() + )) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + let path_with_null = Path::new("/tmp/foo\0bar.bin"); + + assert_eq!( + context.state_save_file(path_with_null, &[]).err(), + CString::new("/tmp/foo\0bar.bin") + .err() + .map(SaveSessionError::NullError) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_load_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let path_with_null = Path::new("/tmp/foo\0bar.bin"); + + assert_eq!( + context.state_load_file(path_with_null, 512).err(), + CString::new("/tmp/foo\0bar.bin") + .err() + .map(LoadSessionError::NullError) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_save_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + let path_with_null = Path::new("/tmp/foo\0bar.bin"); + + assert_eq!( + context.state_seq_save_file(path_with_null, 0, &[]).err(), + CString::new("/tmp/foo\0bar.bin") + .err() + .map(SaveSeqStateError::NullError) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_load_file_refuses_a_path_with_a_nul_byte(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let path_with_null = Path::new("/tmp/foo\0bar.bin"); + + assert_eq!( + context.state_seq_load_file(path_with_null, 0, 512).err(), + CString::new("/tmp/foo\0bar.bin") + .err() + .map(LoadSeqStateError::NullError) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs new file mode 100644 index 000000000..fdcd9a664 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/session_file_token_capacity.rs @@ -0,0 +1,129 @@ +use std::env; +use std::fs; + +use anyhow::Result; + +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::load_seq_state_error::LoadSeqStateError; +use llama_cpp_bindings::context::load_session_error::LoadSessionError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_load_file_with_too_small_a_token_capacity_reports_failed_to_load( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = fixture.model.str_to_token( + "Hello world this is a longer string for more tokens", + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = env::temp_dir().join("llama_test_session_insuf.bin"); + context.state_save_file(&session_path, &tokens)?; + + assert_eq!( + context.state_load_file(&session_path, 1).err(), + Some(LoadSessionError::FailedToLoad) + ); + fs::remove_file(&session_path)?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_load_file_with_too_small_a_token_capacity_reports_failed_to_load( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = fixture.model.str_to_token( + "Hello world this is a longer string for more tokens", + AddBos::Always, + ParseSpecialTokens::Always, + )?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = env::temp_dir().join("llama_test_seq_state_insuf.bin"); + context.state_seq_save_file(&session_path, 0, &tokens)?; + + assert_eq!( + context.state_seq_load_file(&session_path, 0, 1).err(), + Some(LoadSeqStateError::FailedToLoad) + ); + fs::remove_file(&session_path)?; + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/session_files.rs b/llama-cpp-bindings-tests/tests/session_files.rs new file mode 100644 index 000000000..e18c961e6 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/session_files.rs @@ -0,0 +1,317 @@ +use std::env; +use std::fs; + +use anyhow::Result; + +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::load_seq_state_error::LoadSeqStateError; +use llama_cpp_bindings::context::load_session_error::LoadSessionError; +use llama_cpp_bindings::context::save_seq_state_error::SaveSeqStateError; +use llama_cpp_bindings::context::save_session_error::SaveSessionError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_loaded_state_file_restores_the_tokens_it_was_saved_with( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = env::temp_dir().join("llama_test_session.bin"); + context.state_save_file(&session_path, &tokens)?; + + let loaded_tokens = context.state_load_file(&session_path, 512)?; + assert_eq!(loaded_tokens, tokens); + + fs::remove_file(&session_path)?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_loaded_sequence_state_file_restores_its_tokens_and_bytes( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let session_path = env::temp_dir().join("llama_test_seq_state.bin"); + let bytes_written = context.state_seq_save_file(&session_path, 0, &tokens)?; + assert!(bytes_written > 0); + + let loaded = context.state_seq_load_file(&session_path, 0, 512)?; + + assert_eq!(loaded.tokens, tokens); + assert_eq!( + loaded.bytes_read, bytes_written, + "loading must consume exactly the bytes the save produced" + ); + + fs::remove_file(&session_path)?; + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_load_file_reports_a_missing_file_as_failed_to_load( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context + .state_load_file("/nonexistent/session.bin", 512) + .err(), + Some(LoadSessionError::FailedToLoad) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_load_file_reports_a_missing_file_as_failed_to_load( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + assert_eq!( + context + .state_seq_load_file("/nonexistent/seq_state.bin", 0, 512) + .err(), + Some(LoadSeqStateError::FailedToLoad) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_save_file_to_invalid_directory_returns_failed_to_save( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = fixture.build_context()?; + + assert_eq!( + context.state_save_file("/nonexistent_dir/session.bin", &[]), + Err(SaveSessionError::FailedToSave) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_save_file_to_invalid_directory_returns_failed_to_save( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let context = fixture.build_context()?; + + assert_eq!( + context.state_seq_save_file("/nonexistent_dir/seq_state.bin", 0, &[]), + Err(SaveSeqStateError::FailedToSave) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/special_tokens.rs b/llama-cpp-bindings-tests/tests/special_tokens.rs new file mode 100644 index 000000000..a55d2ad41 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/special_tokens.rs @@ -0,0 +1,196 @@ +use anyhow::Context; +use anyhow::Result; + +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn the_eos_token_differs_from_the_bos_token(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let eos = model.token_eos().context("model declares an EOS token")?; + + assert_ne!(model.token_bos(), Some(eos)); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 64, + embeddings = true, +)] +fn model_without_a_bos_token_reports_none(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.token_bos(), None); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_nl_is_the_first_token_of_a_tokenized_newline(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let newline_tokens = model.str_to_token("\n", AddBos::Never, ParseSpecialTokens::Never)?; + + assert_eq!(model.token_nl(), newline_tokens.first().copied()); + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn decoder_only_model_has_no_decoder_start_token(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.decode_start_token(), None); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn chat_model_has_no_separator_token(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.model.token_sep(), None); +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_adds_a_bos_token(fixture: &LlamaFixture<'_>) { + assert!(fixture.model.adds_bos_token()); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_does_not_add_a_bos_token(fixture: &LlamaFixture<'_>) { + assert!(!fixture.model.adds_bos_token()); +} diff --git a/llama-cpp-bindings-tests/tests/state_snapshots.rs b/llama-cpp-bindings-tests/tests/state_snapshots.rs new file mode 100644 index 000000000..dbf8b6aab --- /dev/null +++ b/llama-cpp-bindings-tests/tests/state_snapshots.rs @@ -0,0 +1,206 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::context::llama_state_seq_flags::LlamaStateSeqFlags; +use llama_cpp_bindings::context::state_data_error::StateDataError; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_fresh_context_reports_a_nonempty_state_size(fixture: &LlamaFixture<'_>) -> Result<()> { + let context = fixture.build_context()?; + + assert!(context.state_get_size() > 0); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_set_data_rejects_a_truncated_snapshot(fixture: &LlamaFixture<'_>) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let mut state_data = vec![0u8; context.state_get_size()]; + let bytes_copied = unsafe { context.state_get_data(&mut state_data) }?; + state_data.truncate(bytes_copied / 2); + + assert_eq!( + unsafe { context.state_set_data(&state_data) }, + Err(StateDataError::NothingRestored { + provided_bytes: state_data.len(), + }), + "a truncated snapshot restores nothing, and that must not be reported as success" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_set_data_consumes_exactly_the_bytes_state_get_data_wrote( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let state_size = context.state_get_size(); + let mut state_data = vec![0u8; state_size]; + let bytes_copied = unsafe { context.state_get_data(&mut state_data) }?; + assert!(bytes_copied > 0); + + let bytes_read = unsafe { context.state_set_data(&state_data) }?; + assert_eq!( + bytes_read, bytes_copied, + "restoring the state must consume exactly the bytes the snapshot produced" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn state_seq_set_data_ext_consumes_exactly_the_bytes_state_seq_get_data_ext_wrote( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = fixture.build_context()?; + + let tokens = + fixture + .model + .str_to_token("Hello world", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let flags = LlamaStateSeqFlags::empty(); + let size = context.state_seq_get_size_ext(0, &flags); + let mut buffer = vec![0u8; size]; + let bytes_written = unsafe { context.state_seq_get_data_ext(&mut buffer, 0, &flags) }?; + + assert!(bytes_written > 0); + assert_eq!(bytes_written, size); + + let bytes_read = unsafe { context.state_seq_set_data_ext(&buffer, 0, &flags) }?; + + assert_eq!( + bytes_read, bytes_written, + "restoring the sequence state must consume exactly the bytes the snapshot produced" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/structured_chat_output.rs b/llama-cpp-bindings-tests/tests/structured_chat_output.rs deleted file mode 100644 index 6796f5b60..000000000 --- a/llama-cpp-bindings-tests/tests/structured_chat_output.rs +++ /dev/null @@ -1,2357 +0,0 @@ -use anyhow::Context; -use anyhow::Result; -use anyhow::bail; -use llama_cpp_bindings::BareJsonToolCalls; -use llama_cpp_bindings::ChatMessageParseOutcome; -use llama_cpp_bindings::ChatMessageParser; -use llama_cpp_bindings::GenerationProgress; -use llama_cpp_bindings::MarkerRole; -use llama_cpp_bindings::ParsedChatMessage; -use llama_cpp_bindings::ParsedToolCall; -use llama_cpp_bindings::ParsedToolCallId; -use llama_cpp_bindings::SampledTokenSection; -use llama_cpp_bindings::TokenUsage; -use llama_cpp_bindings::ToolCallArgsShape; -use llama_cpp_bindings::ToolCallArguments; -use llama_cpp_bindings::context::LlamaContext; -use llama_cpp_bindings::llama_batch::LlamaBatch; -use llama_cpp_bindings::model::AddBos; -use llama_cpp_bindings::model::LlamaChatMessage; -use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::model::ParseSpecialTokens; -use llama_cpp_bindings::sampling::LlamaSampler; -use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoop; -use llama_cpp_bindings_tests::classify_sample_loop::ClassifySampleLoopOutcome; -use llama_cpp_bindings_types::BatchCapacity; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; -use serde_json::Value; -use serde_json::json; - -fn parse_partial_reasoning_response( - model: &LlamaModel, - generated: &str, -) -> Result { - let Some(markers) = model.reasoning_markers()? else { - bail!("model must expose reasoning markers"); - }; - let response = if generated.trim_start().starts_with(markers.open.trim()) { - generated.to_owned() - } else { - format!("{}{generated}", markers.open) - }; - let parse_outcome = ChatMessageParser::new(model, "[]")?.parse(&response, true)?; - let ChatMessageParseOutcome::Recognized(parsed) = parse_outcome else { - bail!("model chat template must recognize a partial reasoning response"); - }; - Ok(parsed) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn deepseek_r1_8b_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const DEEPSEEK_R1_8B_THINKING_DISABLED_PROMPT: &str = "\ -<｜User｜>What is 2 + 2?<｜Assistant｜> - - - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - DEEPSEEK_R1_8B_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert!( - !outcome.generated_raw.is_empty(), - "DeepSeek-R1-8B: must generate at least one token" - ); - assert_eq!( - outcome.observed_reasoning, 0, - "DeepSeek-R1-8B thinking-disabled: classifier must not emit any Reasoning token \ - when the prompt closes the think block before generation begins; \ - generated={:?}", - outcome.generated_raw - ); - assert_eq!( - outcome.observed_undeterminable, 0, - "DeepSeek-R1-8B thinking-disabled: prompt-token replay must move section to Content \ - before generation, so no Undeterminable tokens may be emitted; \ - generated={:?}", - outcome.generated_raw - ); - assert_eq!( - usage.reasoning_tokens, 0, - "DeepSeek-R1-8B thinking-disabled: usage.reasoning_tokens must be zero; usage={usage:?}" - ); - assert_eq!( - usage.undeterminable_tokens, 0, - "DeepSeek-R1-8B thinking-disabled: usage.undeterminable_tokens must be zero; usage={usage:?}" - ); - assert!( - outcome.observed_content > 0, - "DeepSeek-R1-8B thinking-disabled: classifier must emit at least one Content token" - ); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content, - "DeepSeek-R1-8B thinking-disabled: completion tokens must equal observed Content tokens" - ); - - for forbidden in FORBIDDEN_MARKERS { - assert!( - !outcome.content_stream.contains(forbidden), - "DeepSeek-R1-8B thinking-disabled: content_stream leaked marker {forbidden:?}; \ - content_stream={:?}", - outcome.content_stream - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn deepseek_r1_8b_classifier_emits_reasoning_for_thinking_enabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const DEEPSEEK_R1_8B_THINKING_PROMPT: &str = "\ -<｜User｜>What is 2 + 2?<｜Assistant｜> -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - DEEPSEEK_R1_8B_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_deepseek_r1_token_counts(&outcome, usage); - assert_deepseek_r1_streams(&outcome, &parsed, MAX_GENERATED_TOKENS, FORBIDDEN_MARKERS); - - Ok(()) -} - -fn assert_deepseek_r1_token_counts(outcome: &ClassifySampleLoopOutcome, usage: &TokenUsage) { - assert!( - !outcome.generated_raw.is_empty(), - "DeepSeek-R1-8B: must generate at least one token" - ); - assert!( - outcome.observed_reasoning > 0, - "DeepSeek-R1-8B: classifier must emit at least one Reasoning token when the prompt \ - opens a block; outcome={outcome:?}", - ); - assert!( - usage.reasoning_tokens > 0, - "DeepSeek-R1-8B: usage.reasoning_tokens must be non-zero when the prompt opens a \ - block; usage was {usage:?}" - ); - assert_eq!( - outcome.observed_undeterminable, 0, - "DeepSeek-R1-8B: prompt-token replay must move section to Reasoning before generation, \ - so no Undeterminable tokens may be emitted; outcome={outcome:?}" - ); - assert_eq!( - usage.undeterminable_tokens, 0, - "DeepSeek-R1-8B: usage.undeterminable_tokens must be zero; usage={usage:?}" - ); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - "DeepSeek-R1-8B: completion tokens must equal observed Content + Reasoning" - ); -} - -fn assert_deepseek_r1_streams( - outcome: &ClassifySampleLoopOutcome, - parsed: &ParsedChatMessage, - max_generated_tokens: i32, - forbidden_markers: &[&str], -) { - assert!( - !parsed.reasoning_content.is_empty(), - "DeepSeek-R1-8B partial response must expose reasoning within {max_generated_tokens} tokens" - ); - assert_eq!( - outcome.reasoning_stream, parsed.reasoning_content, - "DeepSeek-R1-8B: per-token reasoning stream must equal parser-side reasoning_content \ - (any difference means a marker leaked into the user-visible stream)", - ); - assert_eq!( - outcome.content_stream, parsed.content, - "DeepSeek-R1-8B: per-token content stream must equal parser-side content \ - (any difference means a marker leaked into the user-visible stream)", - ); - - for forbidden in forbidden_markers { - assert!( - !outcome.reasoning_stream.contains(forbidden), - "DeepSeek-R1-8B: reasoning_stream leaked marker {forbidden:?}; \ - reasoning_stream={:?}", - outcome.reasoning_stream - ); - assert!( - !outcome.content_stream.contains(forbidden), - "DeepSeek-R1-8B: content_stream leaked marker {forbidden:?}; \ - content_stream={:?}", - outcome.content_stream - ); - } -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_gemma_paired_quote(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const GEMMA_PAIRED_QUOTE_PAYLOAD: &str = - "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}"; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)? - .parse(GEMMA_PAIRED_QUOTE_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "duck-type pass must recognise Gemma paired-quote on a model with no registered \ - template; got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_glm_key_value_tags(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const GLM_KEY_VALUE_PAYLOAD: &str = "get_weather\ -location\ -Paris\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(GLM_KEY_VALUE_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "duck-type pass must recognise GLM key-value tags on a model with no registered \ - template; got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_mistral_bracketed_json(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const MISTRAL_BRACKETED_JSON_PAYLOAD: &str = - r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)? - .parse(MISTRAL_BRACKETED_JSON_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "duck-type pass must recognise Mistral bracketed-JSON on a model with no registered \ - template; got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_duck_types_qwen_xml(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const QWEN_XML_PAYLOAD: &str = "\n\ -\n\ -\n\ -Paris\n\ -\n\ -\n\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(QWEN_XML_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "duck-type pass must recognise Qwen XML on a model with no registered template; \ - got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_recognizes_empty_tool_calls_when_input_is_plain_content_with_tools_requested( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const PLAIN_CONTENT: &str = "Sorry, I cannot help with that."; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(PLAIN_CONTENT, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "plain content with tools requested must produce Recognized (with empty tool_calls); \ - got Unrecognized" - ); - }; - assert!( - parsed.tool_calls.is_empty(), - "expected no tool calls; got {:?}", - parsed.tool_calls - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn deepseek_r1_8b_recognizes_empty_tool_calls_when_tools_not_requested( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const PLAIN_CONTENT: &str = "Hello there."; - - let outcome = ChatMessageParser::new(fixture.model, "[]")?.parse(PLAIN_CONTENT, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("plain content with empty tools array must produce Recognized; got Unrecognized"); - }; - assert!( - parsed.tool_calls.is_empty(), - "expected no tool calls; got {:?}", - parsed.tool_calls - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn gemma4_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const GEMMA4_THINKING_DISABLED_PROMPT: &str = "\ -user\nReply with the single word: four. Do not explain.\n\ -model\n<|channel>thought\n\n"; - - const FORBIDDEN_MARKERS: &[&str] = &["<|channel>thought", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - GEMMA4_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert!( - !outcome.generated_raw.is_empty(), - "Gemma 4 must generate at least one token" - ); - assert_eq!( - outcome.observed_reasoning, 0, - "Gemma 4 thinking-disabled: classifier must not emit any Reasoning token \ - when the prompt closes the thought channel before generation begins; \ - generated={:?}", - outcome.generated_raw - ); - assert_eq!( - outcome.observed_undeterminable, 0, - "Gemma 4 thinking-disabled: prompt-token replay must move section to Content \ - before generation, so no Undeterminable tokens may be emitted; \ - generated={:?}", - outcome.generated_raw - ); - assert_eq!( - usage.reasoning_tokens, 0, - "Gemma 4 thinking-disabled: usage.reasoning_tokens must be zero; usage={usage:?}" - ); - assert_eq!( - usage.undeterminable_tokens, 0, - "Gemma 4 thinking-disabled: usage.undeterminable_tokens must be zero; usage={usage:?}" - ); - assert!( - outcome.observed_content > 0, - "Gemma 4 thinking-disabled: classifier must emit at least one Content token" - ); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content, - "Gemma 4 thinking-disabled: completion tokens must equal observed Content tokens" - ); - - for forbidden in FORBIDDEN_MARKERS { - assert!( - !outcome.content_stream.contains(forbidden), - "Gemma 4 thinking-disabled: content_stream leaked marker {forbidden:?}; \ - content_stream={:?}", - outcome.content_stream - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn gemma4_classifier_emits_reasoning_for_thinking_prompt(fixture: &LlamaFixture<'_>) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const GEMMA4_THINKING_PROMPT: &str = "\ -user\nReply with the single word: four. Do not explain.\n\ -model\n<|channel>thought\n"; - - const FORBIDDEN_MARKERS: &[&str] = &["<|channel>thought", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - GEMMA4_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert!( - !outcome.generated_raw.is_empty(), - "Gemma 4 must generate at least one token" - ); - assert!( - outcome.observed_reasoning > 0, - "Gemma 4 classifier must emit at least one Reasoning token when the model \ - emits a `<|channel>thought` block; outcome={outcome:?}", - ); - assert!( - usage.reasoning_tokens > 0, - "Gemma 4 usage.reasoning_tokens must be non-zero when the model emits a \ - reasoning block; usage was {usage:?}" - ); - assert_eq!( - outcome.observed_undeterminable, 0, - "Gemma 4: classifier must not emit Undeterminable when the model emits a \ - detected `<|channel>thought` marker; outcome={outcome:?}" - ); - assert_eq!( - usage.undeterminable_tokens, 0, - "Gemma 4: usage.undeterminable_tokens must be zero; usage={usage:?}" - ); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - "Gemma 4: completion tokens must equal observed Content + Reasoning" - ); - assert!( - !parsed.reasoning_content.is_empty(), - "Gemma 4 partial response must expose reasoning within {MAX_GENERATED_TOKENS} tokens" - ); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!( - !outcome.reasoning_stream.contains(forbidden), - "Gemma 4: reasoning_stream leaked marker {forbidden:?}; \ - reasoning_stream={:?}", - outcome.reasoning_stream - ); - assert!( - !outcome.content_stream.contains(forbidden), - "Gemma 4: content_stream leaked marker {forbidden:?}; \ - content_stream={:?}", - outcome.content_stream - ); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn gemma4_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const GEMMA4_PAIRED_QUOTE_PAYLOAD: &str = - "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}"; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)? - .parse(GEMMA4_PAIRED_QUOTE_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for Gemma 4 PairedQuote on a Gemma-4 model; got Unrecognized"); - }; - assert_eq!( - parsed.tool_calls.len(), - 1, - "expected one tool call; got {:?}", - parsed.tool_calls - ); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn gemma4_template_format_returns_full_markers(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let template = model - .chat_template(None) - .expect("Gemma 4 chat template must be present"); - let template_str = template.to_str().expect("template must be valid UTF-8"); - assert!( - template_str.contains("<|tool_call>call:"), - "Gemma 4 chat template must contain '<|tool_call>call:' fingerprint; \ - template starts with: {:?}", - &template_str[..template_str.len().min(200)], - ); - - let markers = model - .tool_call_markers()? - .expect("Gemma 4 must produce ToolCallMarkers from its template format"); - - assert_eq!(markers.open, "<|tool_call>call:"); - assert_eq!(markers.close, "}"); - let ToolCallArgsShape::PairedQuote(shape) = markers.args_shape else { - panic!("expected PairedQuote variant, got {:?}", markers.args_shape); - }; - assert_eq!(shape.name_args_separator, "{"); - assert_eq!(shape.value_quote.open, "<|\"|>"); - assert_eq!(shape.value_quote.close, "<|\"|>"); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn glm47_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const GLM47_THINKING_DISABLED_PROMPT: &str = "\ -<|user|> -What is 2 + 2? -<|assistant|> - - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - GLM47_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert_ne!(outcome.generated_raw, ""); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert!(outcome.observed_content > 0); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn glm47_classifier_emits_reasoning_for_thinking_enabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const GLM47_THINKING_PROMPT: &str = "\ -<|user|> -What is 2 + 2? -<|assistant|> - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - GLM47_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(usage.reasoning_tokens > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning - ); - - assert!( - !parsed.reasoning_content.is_empty(), - "GLM-4.7 partial response must expose reasoning within {MAX_GENERATED_TOKENS} tokens" - ); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.reasoning_stream.contains(forbidden)); - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn glm47_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const GLM47_KEY_VALUE_PAYLOAD: &str = "get_weather\ -location\ -Paris\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(GLM47_KEY_VALUE_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "expected Recognized for GLM-4.7 key-value tags on a GLM-4.7-Flash model; got Unrecognized" - ); - }; - assert_eq!(parsed.tool_calls.len(), 1); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn glm47_template_format_returns_full_markers(fixture: &LlamaFixture<'_>) -> Result<()> { - let model = fixture.model; - let template = model - .chat_template(None) - .expect("GLM-4.7 chat template must be present"); - let template_str = template.to_str().expect("template must be valid UTF-8"); - assert!(template_str.contains("")); - - let markers = model - .tool_call_markers()? - .expect("GLM-4.7 must produce ToolCallMarkers from its template format"); - - assert_eq!(markers.open, ""); - assert_eq!(markers.close, ""); - let ToolCallArgsShape::KeyValueXmlTags(shape) = markers.args_shape else { - panic!( - "expected KeyValueXmlTags variant, got {:?}", - markers.args_shape - ); - }; - assert_eq!(shape.key_open, ""); - assert_eq!(shape.key_close, ""); - assert_eq!(shape.value_open, ""); - assert_eq!(shape.value_close, ""); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn mistral3_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const MISTRAL3_THINKING_DISABLED_PROMPT: &str = "\ -[INST]Reply with the single word: four. Do not explain.[/INST][THINK][/THINK]"; - - const FORBIDDEN_MARKERS: &[&str] = &["[THINK]", "[/THINK]"]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - MISTRAL3_THINKING_DISABLED_PROMPT, - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert_ne!(outcome.generated_raw, ""); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert!(outcome.observed_content > 0); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn mistral3_classifier_emits_reasoning_for_thinking_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const MISTRAL3_THINKING_PROMPT: &str = "\ -[SYSTEM_PROMPT]# HOW YOU SHOULD THINK AND ANSWER\n\n\ -First draft your thinking process (inner monologue) until you arrive at a response. \ -Format your response using Markdown, and use LaTeX for any mathematical equations. \ -Write both your thoughts and the response in the same language as the input.\n\n\ -Your thinking process must follow the template below:\ -[THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. \ -Be as casual and as long as you want until you are confident to generate the response \ -to the user.[/THINK]Here, provide a self-contained response.[/SYSTEM_PROMPT]\ -[INST]Reply with the single word: four. Do not explain.[/INST]"; - - const FORBIDDEN_MARKERS: &[&str] = &["[THINK]", "[/THINK]"]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - MISTRAL3_THINKING_PROMPT, - AddBos::Always, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(usage.reasoning_tokens > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - ); - assert_ne!(parsed.reasoning_content, ""); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.reasoning_stream.contains(forbidden)); - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn mistral3_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const MISTRAL3_BRACKETED_JSON_PAYLOAD: &str = - r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)? - .parse(MISTRAL3_BRACKETED_JSON_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "expected Recognized for Mistral 3 BracketedJson on a Mistral-3 model; got Unrecognized" - ); - }; - assert_eq!(parsed.tool_calls.len(), 1); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn qwen35_chat_inference_emits_reasoning_when_template_auto_opens( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let chat_template = model.chat_template(None)?; - let messages = vec![LlamaChatMessage::new( - "user".to_owned(), - "Hello! How are you?".to_owned(), - )?]; - let prompt = model.apply_chat_template(&chat_template, &messages, true, true)?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let tokens = model.str_to_token(&prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let prompt_token_count = u64::try_from(tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: 512, - } - .run()?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(outcome.observed_tool_call, 0); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, prompt_token_count); - assert_eq!(usage.reasoning_tokens, outcome.observed_reasoning); - assert_eq!(usage.undeterminable_tokens, 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn qwen35_shared_reasoning_close_and_tool_call_open_is_one_transition( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let reasoning_markers = fixture - .model - .reasoning_markers()? - .expect("Qwen3.5 must expose reasoning markers"); - let streaming_markers = fixture.model.streaming_markers()?; - - assert!(streaming_markers.iter().any(|marker| { - marker.roles().contains(&MarkerRole::ReasoningOpen) && !marker.tokens().is_empty() - })); - assert_eq!( - streaming_markers - .iter() - .filter(|marker| marker.roles().contains(&MarkerRole::ReasoningClose)) - .count(), - reasoning_markers.closes.len() - ); - - let reasoning_open = streaming_markers - .iter() - .find(|marker| marker.roles().contains(&MarkerRole::ReasoningOpen)) - .expect("Qwen3.5 must expose a reasoning opener") - .tokens() - .to_vec(); - let shared_boundary = streaming_markers - .iter() - .find(|marker| { - marker.roles().contains(&MarkerRole::ReasoningClose) - && marker.roles().contains(&MarkerRole::ToolCallOpen) - }) - .expect("Qwen3.5 must share its tool-call opener with a reasoning close") - .tokens() - .to_vec(); - - let mut classifier = fixture - .model - .sampled_token_classifier(BareJsonToolCalls::Detect)?; - classifier.ingest_prompt_tokens(&reasoning_open); - assert_eq!(classifier.current_section(), SampledTokenSection::Reasoning); - - classifier.ingest_prompt_tokens(&shared_boundary); - assert_eq!(classifier.current_section(), SampledTokenSection::ToolCall); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Ministral-3-3B-Reasoning-2512-GGUF", "Ministral-3-3B-Reasoning-2512-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn mistral3_tool_call_turn_finished_in_the_prompt_leaves_the_classifier_pending( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let mut finished_tool_call_turn = model.str_to_token( - "[TOOL_CALLS]read[ARGS]{\"path\":\"marker.txt\"}", - AddBos::Never, - ParseSpecialTokens::Always, - )?; - - finished_tool_call_turn.push(model.token_eos().context("model declares an EOS token")?); - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - - classifier.ingest_prompt_tokens(&finished_tool_call_turn); - - assert_eq!(classifier.current_section(), SampledTokenSection::Pending); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn qwen35_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const QWEN35_THINKING_DISABLED_PROMPT: &str = "\ -<|im_start|>user -What is 2 + 2?<|im_end|> -<|im_start|>assistant - - - - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - QWEN35_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert_ne!(outcome.generated_raw, ""); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert!(outcome.observed_content > 0); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn qwen35_classifier_emits_reasoning_for_thinking_enabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const QWEN35_THINKING_PROMPT: &str = "\ -<|im_start|>user -What is 2 + 2?<|im_end|> -<|im_start|>assistant - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - QWEN35_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(usage.reasoning_tokens > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - ); - - assert!( - !parsed.reasoning_content.is_empty(), - "Qwen3.5 must close its reasoning block within {MAX_GENERATED_TOKENS} tokens" - ); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.reasoning_stream.contains(forbidden)); - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -fn arguments_as_json(arguments: &ToolCallArguments) -> Result<&Value> { - match arguments { - ToolCallArguments::ValidJson(value) => Ok(value), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson arguments, got InvalidJson: {raw}") - } - } -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_parses_constrained_schema_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const NEGOTIATE_WITH_CAT_TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "negotiate_with_cat", - "description": "Attempt to negotiate with a cat. Outcomes are not guaranteed and may include the silent treatment.", - "parameters": { - "type": "object", - "properties": { - "topic": { - "type": "string", - "description": "What you are trying to negotiate, e.g. 'get off the keyboard' or 'stop knocking things off the table'" - }, - "bribe": { - "type": "string", - "enum": ["tuna", "salmon", "treats", "ear_scritches", "cardboard_box", "none"], - "description": "What you are offering in exchange" - }, - "desperation_level": { - "type": "integer", - "description": "How desperate you are, on a scale from 1 (mildly annoyed human) to 10 (it is 3am)", - "minimum": 1, - "maximum": 10 - } - }, - "required": ["topic"], - "additionalProperties": false - } - } - } -]"#; - - const NEGOTIATE_WITH_CAT_INPUT: &str = "\n\ -\n\ -\n\ -tuna\n\ -\n\ -\n\ -8\n\ -\n\ -\n\ -get off the keyboard\n\ -\n\ -\n\ -"; - - let outcome = ChatMessageParser::new(fixture.model, NEGOTIATE_WITH_CAT_TOOLS_JSON)? - .parse(NEGOTIATE_WITH_CAT_INPUT, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "Qwen 3.5's tool-call payload must be parsed by the wrapper-side duck-type pass; \ - got Unrecognized" - ); - }; - - assert_eq!(parsed.tool_calls.len(), 1); - assert_eq!(parsed.tool_calls[0].name, "negotiate_with_cat"); - assert_eq!( - parsed.tool_calls[0].id, - ParsedToolCallId::NotProvidedByModel - ); - assert_eq!( - arguments_as_json(&parsed.tool_calls[0].arguments)?, - &json!({ - "bribe": "tuna", - "desperation_level": 8, - "topic": "get off the keyboard", - }), - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const QWEN_XML_PAYLOAD: &str = "\n\ -\n\ -\n\ -Paris\n\ -\n\ -\n\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(QWEN_XML_PAYLOAD, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!("expected Recognized for Qwen XML on a Qwen-3.5 model; got Unrecognized"); - }; - assert_eq!(parsed.tool_calls.len(), 1); - assert_eq!(parsed.tool_calls[0].name, "get_weather"); - let location = match &parsed.tool_calls[0].arguments { - ToolCallArguments::ValidJson(value) => value - .get("location") - .and_then(|value| value.as_str()) - .map(str::to_owned), - ToolCallArguments::InvalidJson(raw) => { - bail!("expected ValidJson, got InvalidJson: {raw}"); - } - }; - assert_eq!(location.as_deref(), Some("Paris")); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_parses_partial_tool_call_returns_pending_state(fixture: &LlamaFixture<'_>) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const PARTIAL_QWEN_XML_PAYLOAD: &str = "\n\n) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const TWO_QWEN_XML_PAYLOADS: &str = "\n\ -\n\ -\n\ -Paris\n\ -\n\ -\n\ -\n\ -\n\ -\n\ -\n\ -Berlin\n\ -\n\ -\n\ -"; - - let outcome = - ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(TWO_QWEN_XML_PAYLOADS, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "expected Recognized for two Qwen XML payloads on a Qwen-3.5 model; got Unrecognized" - ); - }; - assert_eq!( - parsed.tool_calls, - vec![ - ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ), - ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Berlin"})), - ), - ] - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 512, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_recognizes_empty_tool_calls_when_input_is_plain_content_with_tools_requested( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const TOOLS_JSON: &str = r#"[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "The city name"} - }, - "required": ["location"] - } - } - } -]"#; - - const PLAIN_CONTENT: &str = "Sorry, I cannot help with that."; - - let outcome = ChatMessageParser::new(fixture.model, TOOLS_JSON)?.parse(PLAIN_CONTENT, false)?; - - let ChatMessageParseOutcome::Recognized(parsed) = outcome else { - bail!( - "Qwen 3.5 with tools requested + plain content must produce Recognized (with empty \ - tool_calls); got Unrecognized" - ); - }; - assert!( - parsed.tool_calls.is_empty(), - "expected no tool calls; got {:?}", - parsed.tool_calls - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 2048, - n_batch = 512, - n_ubatch = 128, -)] -fn qwen36_chat_inference_emits_reasoning_when_template_auto_opens( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - let model = fixture.model; - let backend = fixture.backend; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - let chat_template = model.chat_template(None)?; - let messages = vec![LlamaChatMessage::new( - "user".to_owned(), - "Hello! How are you?".to_owned(), - )?]; - let prompt = model.apply_chat_template(&chat_template, &messages, true, true)?; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let tokens = model.str_to_token(&prompt, AddBos::Always, ParseSpecialTokens::Always)?; - let prompt_token_count = u64::try_from(tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(512)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &tokens, 0, false)?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::greedy()?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: 512, - } - .run()?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(outcome.observed_content > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(outcome.observed_tool_call, 0); - - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - assert_ne!(parsed.content, ""); - - let usage = classifier.into_usage(); - assert_eq!(usage.prompt_tokens, prompt_token_count); - assert_eq!(usage.reasoning_tokens, outcome.observed_reasoning); - assert_eq!(usage.undeterminable_tokens, 0); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn qwen36_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 200; - - const QWEN36_THINKING_DISABLED_PROMPT: &str = "\ -<|im_start|>user -What is 2 + 2?<|im_end|> -<|im_start|>assistant - - - - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - QWEN36_THINKING_DISABLED_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - - assert_ne!(outcome.generated_raw, ""); - assert_eq!(outcome.observed_reasoning, 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.reasoning_tokens, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert!(outcome.observed_content > 0); - assert_eq!(usage.completion_tokens(), outcome.observed_content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 8192, - n_batch = 2048, - n_ubatch = 512, -)] -fn qwen36_classifier_emits_reasoning_for_thinking_enabled_prompt( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const MAX_GENERATED_TOKENS: i32 = 512; - - const QWEN36_THINKING_PROMPT: &str = "\ -<|im_start|>user -What is 2 + 2?<|im_end|> -<|im_start|>assistant - -"; - - const FORBIDDEN_MARKERS: &[&str] = &["", ""]; - - let model = fixture.model; - let backend = fixture.backend; - - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let prompt_tokens = model.str_to_token( - QWEN36_THINKING_PROMPT, - AddBos::Never, - ParseSpecialTokens::Always, - )?; - let prompt_token_count = u64::try_from(prompt_tokens.len())?; - - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2048)?, 1); - classifier.feed_prompt_sequence_to_batch(&mut batch, &prompt_tokens, 0, false)?; - - let mut context = LlamaContext::from_model( - model, - backend, - (*fixture.context_params).into_llama_context_params(), - )?; - - context.decode(&mut batch)?; - - let promoted = classifier.commit_prompt_tokens(); - assert_eq!(promoted, prompt_token_count); - - let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0)?, - LlamaSampler::top_k(40)?, - LlamaSampler::top_p(0.9, 1)?, - LlamaSampler::min_p(0.05, 1)?, - LlamaSampler::temp(0.7)?, - LlamaSampler::dist(0x00C0_FFEE)?, - ])?; - let initial_position = batch.n_tokens(); - let outcome = ClassifySampleLoop { - classifier: &mut classifier, - sampler: &mut sampler, - context: &mut context, - batch: &mut batch, - initial_position, - max_generated_tokens: MAX_GENERATED_TOKENS, - } - .run()?; - - let usage = classifier.usage(); - let parsed = parse_partial_reasoning_response(model, &outcome.generated_raw)?; - - assert_ne!(outcome.generated_raw, ""); - assert!(outcome.observed_reasoning > 0); - assert!(usage.reasoning_tokens > 0); - assert_eq!(outcome.observed_undeterminable, 0); - assert_eq!(usage.undeterminable_tokens, 0); - assert_eq!( - usage.completion_tokens(), - outcome.observed_content + outcome.observed_reasoning, - ); - - assert_ne!(parsed.reasoning_content, ""); - assert_eq!(outcome.reasoning_stream, parsed.reasoning_content); - assert_eq!(outcome.content_stream, parsed.content); - - for forbidden in FORBIDDEN_MARKERS { - assert!(!outcome.reasoning_stream.contains(forbidden)); - assert!(!outcome.content_stream.contains(forbidden)); - } - - Ok(()) -} - -fn visible_text_of_generation_ending_after( - model: &LlamaModel, - generated_text: &str, -) -> Result { - let mut classifier = model.sampled_token_classifier(BareJsonToolCalls::Detect)?; - let mut outcomes = Vec::new(); - - for token in model.str_to_token(generated_text, AddBos::Never, ParseSpecialTokens::Always)? { - assert_eq!( - classifier.ingest(token, &mut outcomes)?, - GenerationProgress::Continues - ); - } - - assert_eq!( - classifier.ingest( - model.token_eos().context("model declares an EOS token")?, - &mut outcomes - )?, - GenerationProgress::Ended - ); - - Ok(outcomes - .iter() - .map(|outcome| outcome.piece.visible()) - .collect()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_classifier_ends_generation_without_emitting_the_end_of_generation_token( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const GENERATED_TEXT: &str = "The answer is four."; - - assert_eq!( - visible_text_of_generation_ending_after(fixture.model, GENERATED_TEXT)?, - GENERATED_TEXT - ); - - Ok(()) -} - -#[llama_test( - model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, - load_mode = Mmap, - n_ctx = 256, - n_batch = 128, - n_ubatch = 64, -)] -fn qwen35_classifier_releases_a_held_json_prefix_when_generation_ends( - fixture: &LlamaFixture<'_>, -) -> Result<()> { - const GENERATED_TEXT: &str = r#"{"answer": 4"#; - - assert_eq!( - visible_text_of_generation_ending_after(fixture.model, GENERATED_TEXT)?, - GENERATED_TEXT - ); - - Ok(()) -} diff --git a/llama-cpp-bindings-tests/tests/token_attributes.rs b/llama-cpp-bindings-tests/tests/token_attributes.rs new file mode 100644 index 000000000..f40041fed --- /dev/null +++ b/llama-cpp-bindings-tests/tests/token_attributes.rs @@ -0,0 +1,49 @@ +use anyhow::Context; +use anyhow::Result; + +use llama_cpp_bindings::llama_token_attrs::LlamaTokenAttrs; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn the_eos_token_carries_only_the_control_attribute(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let eos = model.token_eos().context("model declares an EOS token")?; + + assert_eq!( + model.token_attr(eos)?, + LlamaTokenAttrs::try_from(llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_CONTROL)? + ); + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/token_candidates.rs b/llama-cpp-bindings-tests/tests/token_candidates.rs new file mode 100644 index 000000000..a3a3fcfbd --- /dev/null +++ b/llama-cpp-bindings-tests/tests/token_candidates.rs @@ -0,0 +1,83 @@ +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_bindings::token::data::LlamaTokenData; +use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn a_token_data_array_holds_one_candidate_per_vocabulary_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?; + let last_index = i32::try_from(tokens.len() - 1)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let token_data_array = context.token_data_array_ith(last_index)?; + + assert_eq!( + token_data_array.data.len(), + usize::try_from(fixture.model.n_vocab())? + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_refilled_candidate_array_matches_a_freshly_built_one( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let prompt_tokens = model.str_to_token("Hello", AddBos::Never, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&prompt_tokens, 0, false)?; + context.decode(&batch)?; + + let last_index = batch.n_tokens() - 1; + let mut reused_candidates = LlamaTokenDataArray::new( + vec![LlamaTokenData::new(LlamaToken::new(7), 1.0, 0.5)], + true, + ); + reused_candidates.selected = Some(0); + + context.fill_token_data_array_ith(last_index, &mut reused_candidates)?; + + assert_eq!(reused_candidates, context.token_data_array_ith(last_index)?); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tokenization.rs b/llama-cpp-bindings-tests/tests/tokenization.rs new file mode 100644 index 000000000..12f57246b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/tokenization.rs @@ -0,0 +1,180 @@ +use anyhow::Result; + +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn the_first_token_of_tokenized_text_decodes_to_its_first_word( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let tokens = model.str_to_token("hello world", AddBos::Never, ParseSpecialTokens::Always)?; + assert_ne!(tokens, Vec::::new()); + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let piece = + model.token_to_piece(&SampledToken::Content(tokens[0]), &mut decoder, false, None)?; + + assert_eq!(piece, "hello"); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn str_to_token_prepends_bos_for_a_model_that_adds_it(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + let tokens_without_bos = + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)?; + let expected = model + .token_bos() + .into_iter() + .chain(tokens_without_bos) + .collect::>(); + + assert!(model.adds_bos_token()); + assert_eq!( + model.str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?, + expected + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn str_to_token_adds_nothing_for_a_model_that_adds_no_bos( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + + assert!(!model.adds_bos_token()); + assert_eq!( + model.str_to_token("hello", AddBos::Always, ParseSpecialTokens::Always)?, + model.str_to_token("hello", AddBos::Never, ParseSpecialTokens::Always)? + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn str_to_token_regrows_its_buffer_for_more_tokens_than_it_estimated( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let many_numbers = (0_u32..2000) + .map(|number| number.to_string()) + .collect::>() + .join(" "); + + let tokens = + fixture + .model + .str_to_token(&many_numbers, AddBos::Always, ParseSpecialTokens::Always)?; + + assert!(tokens.len() > many_numbers.len() / 2); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 512, +)] +fn tokenizing_without_special_token_parsing_keeps_control_markup_as_text( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let markup = "<|fim_prefix|>"; + let parsed = fixture + .model + .str_to_token(markup, AddBos::Never, ParseSpecialTokens::Always)?; + let plain = fixture + .model + .str_to_token(markup, AddBos::Never, ParseSpecialTokens::Never)?; + + assert_eq!(parsed.len(), 1); + assert!(plain.len() > 1); + assert!(!plain.contains(&parsed[0])); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tool_call_classification.rs b/llama-cpp-bindings-tests/tests/tool_call_classification.rs new file mode 100644 index 000000000..b9965ff11 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/tool_call_classification.rs @@ -0,0 +1,77 @@ +use anyhow::Result; + +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::ingest_outcome::IngestOutcome; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::ingested_generation::ingested_generation; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const REASONING_BEFORE_THE_CALL: &str = "Let me check the weather.\n\n\n"; +const QWEN_XML_PARIS_WEATHER_CALL: &str = "\n\ +\n\ +\n\ +Paris\n\ +\n\ +\n\ +"; + +fn raw_text_where(outcomes: &[IngestOutcome], belongs: impl Fn(&SampledToken) -> bool) -> String { + outcomes + .iter() + .filter(|outcome| belongs(&outcome.sampled_token)) + .map(|outcome| outcome.piece.raw()) + .collect() +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen35_reasoning_closed_by_a_tool_call_is_attributed_to_each_section( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + "What is the weather in Paris?", + GET_WEATHER_TOOLS_JSON, + true, + )?; + let answer = format!("{REASONING_BEFORE_THE_CALL}{QWEN_XML_PARIS_WEATHER_CALL}"); + let mut classifier = SampledTokenClassifier::for_chat_session(&session); + let outcomes = ingested_generation(&mut classifier, fixture.model, &answer); + let parsed = recognized_answer(&session, &answer, ChatOutputCompleteness::Finished); + + assert_eq!( + raw_text_where(&outcomes, |token| matches!( + token, + SampledToken::Reasoning(_) + )), + REASONING_BEFORE_THE_CALL + ); + assert_eq!( + raw_text_where(&outcomes, |token| matches!( + token, + SampledToken::ToolCall(_) + )), + QWEN_XML_PARIS_WEATHER_CALL + ); + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome.sampled_token, SampledToken::Reasoning(_))) + .map(|outcome| outcome.piece.visible()) + .collect::(), + parsed.reasoning_content + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tool_call_grammar.rs b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs new file mode 100644 index 000000000..62b31e3d0 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/tool_call_grammar.rs @@ -0,0 +1,113 @@ +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings_tests::chat_answer_generation::ChatAnswerGeneration; +use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_tests::weather_chat_session::weather_chat_session; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const MAX_TOOL_CALL_TOKENS: i32 = 128; + +fn generated_tool_call_answer( + fixture: &LlamaFixture<'_>, + session: &ChatSession<'_>, +) -> Result { + let grammar_sampler = session + .grammar_sampler()? + .context("a tool request must constrain generation with a grammar")?; + let mut sampler = LlamaSampler::chain([grammar_sampler, LlamaSampler::greedy()?])?; + let answer = ChatAnswerGeneration { + fixture, + session, + sampler: &mut sampler, + max_generated_tokens: MAX_TOOL_CALL_TOKENS, + } + .run(); + + Ok(answer.outcome.generated.text().to_owned()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn a_chat_session_without_tools_needs_no_grammar_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + + assert!(session.grammar_sampler()?.is_none()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn a_tool_choice_of_none_needs_no_grammar_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = weather_chat_session(fixture.model, ChatToolChoice::None); + + assert!(session.grammar_sampler()?.is_none()); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen36_required_tool_call_grammar_constrains_the_answer_to_a_tool_call( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = weather_chat_session(fixture.model, ChatToolChoice::Required); + let answer = generated_tool_call_answer(fixture, &session)?; + + assert!(!session.grammar_is_lazy()); + assert_eq!( + recognized_answer(&session, &answer, ChatOutputCompleteness::Finished).tool_calls, + [paris_weather_tool_call()] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 2048, + n_batch = 512, + n_ubatch = 128, +)] +fn qwen36_automatic_tool_call_grammar_constrains_the_tool_call_after_its_trigger( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = weather_chat_session(fixture.model, ChatToolChoice::Auto); + let answer = generated_tool_call_answer(fixture, &session)?; + + assert!(session.grammar_is_lazy()); + assert_eq!( + recognized_answer(&session, &answer, ChatOutputCompleteness::Finished).tool_calls, + [paris_weather_tool_call()] + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/tool_call_parsing.rs b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs new file mode 100644 index 000000000..cac715f2c --- /dev/null +++ b/llama-cpp-bindings-tests/tests/tool_call_parsing.rs @@ -0,0 +1,321 @@ +use anyhow::Result; +use serde_json::json; + +use llama_cpp_bindings::chat_assistant_turn::ChatAssistantTurn; +use llama_cpp_bindings::chat_output_completeness::ChatOutputCompleteness; +use llama_cpp_bindings::chat_session::ChatSession; +use llama_cpp_bindings::chat_session_options::ChatSessionOptions; +use llama_cpp_bindings::chat_tool_choice::ChatToolChoice; +use llama_cpp_bindings::model::llama_chat_message::LlamaChatMessage; +use llama_cpp_bindings_tests::get_weather_tools_json::GET_WEATHER_TOOLS_JSON; +use llama_cpp_bindings_tests::paris_weather_tool_call::paris_weather_tool_call; +use llama_cpp_bindings_tests::recognized_answer::recognized_answer; +use llama_cpp_bindings_tests::user_turn_chat_session::user_turn_chat_session; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +const WEATHER_QUESTION: &str = "What is the weather in Paris?"; +const PLAIN_CONTENT_REFUSAL: &str = "Sorry, I cannot help with that."; + +const NEGOTIATE_WITH_CAT_TOOLS_JSON: &str = r#"[ + { + "type": "function", + "function": { + "name": "negotiate_with_cat", + "description": "Attempt to negotiate with a cat. Outcomes are not guaranteed and may include the silent treatment.", + "parameters": { + "type": "object", + "properties": { + "topic": { + "type": "string", + "description": "What you are trying to negotiate, e.g. 'get off the keyboard' or 'stop knocking things off the table'" + }, + "bribe": { + "type": "string", + "enum": ["tuna", "salmon", "treats", "ear_scritches", "cardboard_box", "none"], + "description": "What you are offering in exchange" + }, + "desperation_level": { + "type": "integer", + "description": "How desperate you are, on a scale from 1 (mildly annoyed human) to 10 (it is 3am)", + "minimum": 1, + "maximum": 10 + } + }, + "required": ["topic"], + "additionalProperties": false + } + } + } +]"#; + +const QWEN_XML_PARIS_WEATHER_CALL: &str = "\n\ +\n\ +\n\ +Paris\n\ +\n\ +\n\ +"; + +fn assert_parses_to_paris_weather_call(fixture: &LlamaFixture<'_>, answer: &str) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + WEATHER_QUESTION, + GET_WEATHER_TOOLS_JSON, + false, + )?; + let parsed = recognized_answer(&session, answer, ChatOutputCompleteness::Finished); + + assert_eq!(parsed.tool_calls, [paris_weather_tool_call()]); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn gemma4_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_parses_to_paris_weather_call( + fixture, + "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}", + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn glm47_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_parses_to_paris_weather_call( + fixture, + "get_weatherlocationParis", + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn mistral3_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_parses_to_paris_weather_call( + fixture, + r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#, + ) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_parses_its_tool_call_format(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_parses_to_paris_weather_call(fixture, QWEN_XML_PARIS_WEATHER_CALL) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_parses_tool_call_arguments_into_their_schema_types( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + "Get the cat off the keyboard.", + NEGOTIATE_WITH_CAT_TOOLS_JSON, + false, + )?; + let parsed = recognized_answer( + &session, + "\n\ +\n\ +\n\ +get off the keyboard\n\ +\n\ +\n\ +tuna\n\ +\n\ +\n\ +8\n\ +\n\ +\n\ +", + ChatOutputCompleteness::Finished, + ); + + assert_eq!( + parsed.tool_calls, + [ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "negotiate_with_cat".to_owned(), + ToolCallArguments::ValidJson(json!({ + "bribe": "tuna", + "desperation_level": 8, + "topic": "get off the keyboard", + })), + )] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_parses_a_tool_call_cut_inside_its_arguments_as_a_named_call( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + WEATHER_QUESTION, + GET_WEATHER_TOOLS_JSON, + false, + )?; + let parsed = recognized_answer( + &session, + "\n\n>(), + ["get_weather"] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn qwen35_parses_consecutive_tool_calls_in_order(fixture: &LlamaFixture<'_>) -> Result<()> { + let session = ChatSession::new( + fixture.model, + fixture.model.chat_templates()?, + &[LlamaChatMessage::new( + "user".to_owned(), + "What is the weather in Paris and in Berlin?".to_owned(), + )?], + GET_WEATHER_TOOLS_JSON, + &ChatSessionOptions { + assistant_turn: ChatAssistantTurn::New, + enable_thinking: false, + parallel_tool_calls: true, + tool_choice: ChatToolChoice::Auto, + }, + )?; + let parsed = recognized_answer( + &session, + &format!( + "{QWEN_XML_PARIS_WEATHER_CALL}\n{}", + QWEN_XML_PARIS_WEATHER_CALL.replace("Paris", "Berlin") + ), + ChatOutputCompleteness::Finished, + ); + + assert_eq!( + parsed.tool_calls, + [ + paris_weather_tool_call(), + ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "get_weather".to_owned(), + ToolCallArguments::ValidJson(json!({"location": "Berlin"})), + ), + ] + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn plain_content_answering_a_tool_request_parses_without_tool_calls( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session( + fixture.model, + WEATHER_QUESTION, + GET_WEATHER_TOOLS_JSON, + false, + )?; + let parsed = recognized_answer( + &session, + PLAIN_CONTENT_REFUSAL, + ChatOutputCompleteness::Finished, + ); + + assert_eq!(parsed.tool_calls, []); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn deepseek_r1_8b_plain_content_without_tools_parses_without_tool_calls( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let session = user_turn_chat_session(fixture.model, "Hello!", "[]", false)?; + let parsed = recognized_answer(&session, "Hello there.", ChatOutputCompleteness::Finished); + + assert_eq!(parsed.tool_calls, []); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs new file mode 100644 index 000000000..98fa1bbe5 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/unconstrained_token_sampling.rs @@ -0,0 +1,222 @@ +use anyhow::Context as _; +use anyhow::Result; + +use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::context::context_limits::ContextLimits; +use llama_cpp_bindings::generation_progress::GenerationProgress; +use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::model::add_bos::AddBos; +use llama_cpp_bindings::model::parse_special_tokens::ParseSpecialTokens; +use llama_cpp_bindings::sampled_token::SampledToken; +use llama_cpp_bindings::sampled_token_classifier::SampledTokenClassifier; +use llama_cpp_bindings::sampling::LlamaSampler; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +fn most_likely_token(context: &LlamaContext<'_>, token_index: i32) -> Result { + Ok(context + .candidates_ith(token_index)? + .reduce(|most_likely, candidate| { + if candidate.logit() > most_likely.logit() { + candidate + } else { + most_likely + } + }) + .context("a decoded row holds a candidate for every vocabulary token")? + .id()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn sampling_without_a_grammar_counts_every_sampled_token_as_completion( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let mut context = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let prompt = + "<|im_start|>user\nSay hello<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + let tokens = model.str_to_token(prompt, AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + + batch.add_sequence(&tokens, 0, false)?; + + context.decode(&batch)?; + + let mut sampler = LlamaSampler::chain([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; + + let mut classifier = SampledTokenClassifier::undeterminable(model); + let mut outcomes = Vec::new(); + let mut sampled_count: u64 = 0; + let mut ended = false; + + for (position, _) in (batch.n_tokens()..).zip(0..5) { + let turn = classifier.sample(&mut sampler, &context, -1, &mut outcomes)?; + + if turn.progress == GenerationProgress::Ended { + ended = true; + + break; + } + + sampled_count += 1; + + let raw_as_sampled = SampledToken::Content(turn.token); + + batch.clear(); + batch.add(&raw_as_sampled, position, &[0], true)?; + + context.decode(&batch)?; + } + + classifier.finish(&mut outcomes)?; + + assert!( + sampled_count > 0, + "Should produce at least one token without grammar" + ); + assert_eq!( + classifier.into_usage().completion_tokens(), + sampled_count + u64::from(ended), + "completion_tokens must count every sample, the end of generation included" + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn a_greedy_sampler_selects_the_most_likely_candidate_of_a_decoded_row( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("Hi", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + + let last_index = batch.n_tokens() - 1; + let mut data_array = context.token_data_array_ith(last_index)?; + let sampler = LlamaSampler::greedy()?; + sampler.apply(&mut data_array)?; + + assert_eq!( + data_array.selected_token(), + Some(most_likely_token(&context, last_index)?) + ); + + Ok(()) +} + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 256, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, +)] +fn sampling_the_last_decoded_row_with_a_greedy_chain_yields_its_most_likely_token( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mut context = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let tokens = fixture + .model + .str_to_token("Hello", AddBos::Always, ParseSpecialTokens::Always)?; + let mut batch = LlamaBatch::new(ContextLimits::of(&context)); + batch.add_sequence(&tokens, 0, false)?; + context.decode(&batch)?; + let mut sampler = LlamaSampler::chain([LlamaSampler::temp(0.8)?, LlamaSampler::greedy()?])?; + let last_index = batch.n_tokens() - 1; + + assert_eq!( + sampler.sample(&context, last_index)?, + most_likely_token(&context, last_index)? + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs b/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs new file mode 100644 index 000000000..a363955e1 --- /dev/null +++ b/llama-cpp-bindings-tests/tests/vocabulary_bounds.rs @@ -0,0 +1,61 @@ +use llama_cpp_bindings::error::token_attr_error::TokenAttrError; +use llama_cpp_bindings::error::token_outside_vocabulary::TokenOutsideVocabulary; +use llama_cpp_bindings::error::token_to_string_error::TokenToStringError; +use llama_cpp_bindings::token::LlamaToken; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_to_piece_bytes_rejects_tokens_outside_the_vocabulary(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let n_vocab = model.n_vocab(); + + assert_eq!( + [LlamaToken::new(-1), LlamaToken::new(n_vocab)] + .map(|token| model.token_to_piece_bytes(token, 8, true, None)), + [ + Err(TokenToStringError::TokenOutsideVocabulary( + TokenOutsideVocabulary { + token: LlamaToken::new(-1), + n_vocab, + } + )), + Err(TokenToStringError::TokenOutsideVocabulary( + TokenOutsideVocabulary { + token: LlamaToken::new(n_vocab), + n_vocab, + } + )), + ] + ); +} + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn token_attr_rejects_tokens_outside_the_vocabulary(fixture: &LlamaFixture<'_>) { + let model = fixture.model; + let n_vocab = model.n_vocab(); + + assert_eq!( + model.token_attr(LlamaToken::new(n_vocab)).err(), + Some(TokenAttrError::TokenOutsideVocabulary( + TokenOutsideVocabulary { + token: LlamaToken::new(n_vocab), + n_vocab, + } + )) + ); +} diff --git a/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs new file mode 100644 index 000000000..05b72702b --- /dev/null +++ b/llama-cpp-bindings-tests/tests/vocabulary_enumeration.rs @@ -0,0 +1,31 @@ +use anyhow::Result; + +use llama_cpp_bindings::model::vocabulary_token::VocabularyToken; +use llama_cpp_bindings_tests::single_token::single_token; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn the_vocabulary_enumerates_each_token_at_its_id_with_its_piece( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let model = fixture.model; + let hello = single_token(model, "hello"); + + assert_eq!( + model.tokens(false).nth(usize::try_from(hello.0)?), + Some(Ok(VocabularyToken { + token: hello, + piece: "hello".to_owned() + })) + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/vocabulary_type.rs b/llama-cpp-bindings-tests/tests/vocabulary_type.rs new file mode 100644 index 000000000..59ab571ca --- /dev/null +++ b/llama-cpp-bindings-tests/tests/vocabulary_type.rs @@ -0,0 +1,43 @@ +use anyhow::Result; + +use llama_cpp_bindings::model::vocab_type::VocabType; +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness_macros::llama_test; + +#[llama_test( + model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = -1, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, +)] +fn default_models_use_a_byte_pair_encoding_vocabulary(fixture: &LlamaFixture<'_>) -> Result<()> { + assert_eq!(fixture.model.vocab_type()?, VocabType::Bpe); + + Ok(()) +} diff --git a/llama-cpp-bindings-types/src/batch_capacity.rs b/llama-cpp-bindings-types/src/batch_capacity.rs deleted file mode 100644 index 7c4efbf81..000000000 --- a/llama-cpp-bindings-types/src/batch_capacity.rs +++ /dev/null @@ -1,109 +0,0 @@ -use std::num::NonZeroU16; -use std::num::NonZeroU32; - -use crate::batch_capacity_error::BatchCapacityError; - -const MAX_TOKENS: u32 = i32::MAX.unsigned_abs(); - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct BatchCapacity(NonZeroU32); - -impl BatchCapacity { - #[must_use] - pub fn at_most(self, tokens: NonZeroU32) -> Self { - Self(self.0.min(tokens)) - } - - #[must_use] - pub const fn tokens(self) -> NonZeroU32 { - self.0 - } - - #[must_use] - pub const fn tokens_i32(self) -> i32 { - self.0.get().cast_signed() - } - - #[must_use] - pub const fn tokens_usize(self) -> usize { - self.0.get() as usize - } -} - -impl From for BatchCapacity { - fn from(tokens: NonZeroU16) -> Self { - Self(NonZeroU32::from(tokens)) - } -} - -impl TryFrom for BatchCapacity { - type Error = BatchCapacityError; - - fn try_from(tokens: u32) -> Result { - if tokens > MAX_TOKENS { - return Err(BatchCapacityError::TooLarge { - tokens, - max_tokens: MAX_TOKENS, - }); - } - - NonZeroU32::new(tokens) - .map(Self) - .ok_or(BatchCapacityError::Zero) - } -} - -#[cfg(test)] -mod tests { - use std::num::NonZeroU16; - use std::num::NonZeroU32; - - use super::BatchCapacity; - use crate::batch_capacity_error::BatchCapacityError; - - #[test] - fn rejects_a_batch_without_tokens() { - assert_eq!(BatchCapacity::try_from(0), Err(BatchCapacityError::Zero)); - } - - #[test] - fn rejects_a_batch_llama_cpp_cannot_address() { - assert_eq!( - BatchCapacity::try_from(u32::MAX), - Err(BatchCapacityError::TooLarge { - tokens: u32::MAX, - max_tokens: i32::MAX.unsigned_abs(), - }) - ); - } - - #[test] - fn the_largest_batch_keeps_its_size_in_every_width() { - let capacity = BatchCapacity::try_from(i32::MAX.unsigned_abs()).unwrap(); - - assert_eq!(capacity.tokens().get(), i32::MAX.unsigned_abs()); - assert_eq!(capacity.tokens_i32(), i32::MAX); - assert_eq!(capacity.tokens_usize(), i32::MAX.unsigned_abs() as usize); - } - - #[test] - fn a_batch_never_exceeds_the_token_bound_it_is_capped_at() { - let capacity = BatchCapacity::try_from(2048).unwrap(); - - assert_eq!( - [ - capacity.at_most(NonZeroU32::new(512).unwrap()), - capacity.at_most(NonZeroU32::MAX), - ], - [BatchCapacity::try_from(512).unwrap(), capacity] - ); - } - - #[test] - fn every_nonzero_sixteen_bit_size_is_a_valid_batch() { - assert_eq!( - BatchCapacity::from(NonZeroU16::MAX), - BatchCapacity::try_from(u32::from(u16::MAX)).unwrap() - ); - } -} diff --git a/llama-cpp-bindings-types/src/batch_capacity_error.rs b/llama-cpp-bindings-types/src/batch_capacity_error.rs deleted file mode 100644 index 4eb5f857d..000000000 --- a/llama-cpp-bindings-types/src/batch_capacity_error.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum BatchCapacityError { - #[error("a batch must hold at least one token")] - Zero, - #[error("a batch of {tokens} tokens exceeds the {max_tokens} tokens llama.cpp can address")] - TooLarge { tokens: u32, max_tokens: u32 }, -} diff --git a/llama-cpp-bindings-types/src/bracketed_json_shape.rs b/llama-cpp-bindings-types/src/bracketed_json_shape.rs deleted file mode 100644 index 51b18f4b8..000000000 --- a/llama-cpp-bindings-types/src/bracketed_json_shape.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct BracketedJsonShape { - pub name_args_separator: String, -} diff --git a/llama-cpp-bindings-types/src/invalid_arguments_json.rs b/llama-cpp-bindings-types/src/invalid_arguments_json.rs new file mode 100644 index 000000000..2e93fab55 --- /dev/null +++ b/llama-cpp-bindings-types/src/invalid_arguments_json.rs @@ -0,0 +1,21 @@ +use serde::Deserialize; +use serde::Serialize; + +/// Why tool call arguments are not JSON: the parser's message and the 1-based line and column +/// where it stopped. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct InvalidArgumentsJson { + pub message: String, + pub line: usize, + pub column: usize, +} + +impl From for InvalidArgumentsJson { + fn from(error: serde_json::Error) -> Self { + Self { + message: error.to_string(), + line: error.line(), + column: error.column(), + } + } +} diff --git a/llama-cpp-bindings-types/src/json_object_shape.rs b/llama-cpp-bindings-types/src/json_object_shape.rs deleted file mode 100644 index b20a5e204..000000000 --- a/llama-cpp-bindings-types/src/json_object_shape.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct JsonObjectShape { - pub name_field: String, - pub arguments_field: String, -} diff --git a/llama-cpp-bindings-types/src/key_value_xml_tags_shape.rs b/llama-cpp-bindings-types/src/key_value_xml_tags_shape.rs deleted file mode 100644 index 220e94b66..000000000 --- a/llama-cpp-bindings-types/src/key_value_xml_tags_shape.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct KeyValueXmlTagsShape { - pub key_open: String, - pub key_close: String, - pub value_open: String, - pub value_close: String, -} diff --git a/llama-cpp-bindings-types/src/lib.rs b/llama-cpp-bindings-types/src/lib.rs index a35133e72..f6354c6ac 100644 --- a/llama-cpp-bindings-types/src/lib.rs +++ b/llama-cpp-bindings-types/src/lib.rs @@ -3,42 +3,12 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] -pub mod batch_capacity; -pub mod batch_capacity_error; -pub mod bracketed_json_shape; -pub mod json_object_shape; -pub mod key_value_xml_tags_shape; +pub mod invalid_arguments_json; pub mod model_provided_tool_call_id; pub mod model_provided_tool_call_id_error; -pub mod paired_quote_shape; pub mod parsed_chat_message; pub mod parsed_tool_call; pub mod parsed_tool_call_id; -pub mod reasoning_markers; pub mod token_usage; pub mod token_usage_error; -pub mod tool_call_args_shape; pub mod tool_call_arguments; -pub mod tool_call_markers; -pub mod tool_call_value_quote; -pub mod xml_tags_shape; - -pub use batch_capacity::BatchCapacity; -pub use batch_capacity_error::BatchCapacityError; -pub use bracketed_json_shape::BracketedJsonShape; -pub use json_object_shape::JsonObjectShape; -pub use key_value_xml_tags_shape::KeyValueXmlTagsShape; -pub use model_provided_tool_call_id::ModelProvidedToolCallId; -pub use model_provided_tool_call_id_error::ModelProvidedToolCallIdError; -pub use paired_quote_shape::PairedQuoteShape; -pub use parsed_chat_message::ParsedChatMessage; -pub use parsed_tool_call::ParsedToolCall; -pub use parsed_tool_call_id::ParsedToolCallId; -pub use reasoning_markers::ReasoningMarkers; -pub use token_usage::TokenUsage; -pub use token_usage_error::TokenUsageError; -pub use tool_call_args_shape::ToolCallArgsShape; -pub use tool_call_arguments::ToolCallArguments; -pub use tool_call_markers::ToolCallMarkers; -pub use tool_call_value_quote::ToolCallValueQuote; -pub use xml_tags_shape::XmlTagsShape; diff --git a/llama-cpp-bindings-types/src/paired_quote_shape.rs b/llama-cpp-bindings-types/src/paired_quote_shape.rs deleted file mode 100644 index 1126d3ae6..000000000 --- a/llama-cpp-bindings-types/src/paired_quote_shape.rs +++ /dev/null @@ -1,7 +0,0 @@ -use crate::tool_call_value_quote::ToolCallValueQuote; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct PairedQuoteShape { - pub name_args_separator: String, - pub value_quote: ToolCallValueQuote, -} diff --git a/llama-cpp-bindings-types/src/parsed_chat_message.rs b/llama-cpp-bindings-types/src/parsed_chat_message.rs index 0e128309e..de2657e04 100644 --- a/llama-cpp-bindings-types/src/parsed_chat_message.rs +++ b/llama-cpp-bindings-types/src/parsed_chat_message.rs @@ -75,20 +75,24 @@ mod tests { } #[test] - fn message_with_all_three_fields_populated_is_not_empty() { - let parsed = ParsedChatMessage::new( - "hello".to_owned(), - "thinking".to_owned(), - vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "tool".to_owned(), - ToolCallArguments::ValidJson(json!({})), - )], + fn new_places_each_argument_in_its_field() { + let tool_call = ParsedToolCall::new( + ParsedToolCallId::NotProvidedByModel, + "tool".to_owned(), + ToolCallArguments::ValidJson(json!({})), ); - assert!(!parsed.is_empty()); - assert_eq!(parsed.content, "hello"); - assert_eq!(parsed.reasoning_content, "thinking"); - assert_eq!(parsed.tool_calls.len(), 1); + assert_eq!( + ParsedChatMessage::new( + "hello".to_owned(), + "thinking".to_owned(), + vec![tool_call.clone()] + ), + ParsedChatMessage { + content: "hello".to_owned(), + reasoning_content: "thinking".to_owned(), + tool_calls: vec![tool_call], + } + ); } } diff --git a/llama-cpp-bindings-types/src/reasoning_markers.rs b/llama-cpp-bindings-types/src/reasoning_markers.rs deleted file mode 100644 index b29636be8..000000000 --- a/llama-cpp-bindings-types/src/reasoning_markers.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ReasoningMarkers { - pub open: String, - pub closes: Vec, -} diff --git a/llama-cpp-bindings-types/src/token_usage.rs b/llama-cpp-bindings-types/src/token_usage.rs index 6cc1ac6ad..ecd580208 100644 --- a/llama-cpp-bindings-types/src/token_usage.rs +++ b/llama-cpp-bindings-types/src/token_usage.rs @@ -154,9 +154,44 @@ impl<'usage> Sum<&'usage Self> for TokenUsage { #[cfg(test)] mod tests { + use std::iter; + use super::TokenUsage; use super::TokenUsageError; + const LEFT_USAGE: TokenUsage = TokenUsage { + prompt_tokens: 20, + cached_prompt_tokens: 2, + input_image_tokens: 3, + input_audio_tokens: 4, + content_tokens: 5, + reasoning_tokens: 6, + tool_call_tokens: 7, + undeterminable_tokens: 8, + }; + + const RIGHT_USAGE: TokenUsage = TokenUsage { + prompt_tokens: 200, + cached_prompt_tokens: 20, + input_image_tokens: 30, + input_audio_tokens: 40, + content_tokens: 50, + reasoning_tokens: 60, + tool_call_tokens: 70, + undeterminable_tokens: 80, + }; + + const COMBINED_USAGE: TokenUsage = TokenUsage { + prompt_tokens: 220, + cached_prompt_tokens: 22, + input_image_tokens: 33, + input_audio_tokens: 44, + content_tokens: 55, + reasoning_tokens: 66, + tool_call_tokens: 77, + undeterminable_tokens: 88, + }; + #[test] fn new_starts_with_all_counters_at_zero() { let usage = TokenUsage::new(); @@ -215,7 +250,7 @@ mod tests { } #[test] - fn record_cached_equal_to_prompt_succeeds() { + fn caching_every_prompt_token_is_accepted() { let mut usage = TokenUsage::new(); usage.record_prompt_tokens(5); usage @@ -335,48 +370,8 @@ mod tests { } #[test] - fn add_combines_field_by_field() { - let mut left = TokenUsage::new(); - left.record_prompt_tokens(2); - left.record_cached_prompt_tokens(1) - .expect("1 cached <= 2 prompt is valid"); - left.record_content_token(); - left.record_reasoning_token(); - left.record_tool_call_token(); - left.record_undeterminable_token(); - - let mut right = TokenUsage::new(); - right.record_prompt_tokens(5); - right - .record_cached_prompt_tokens(2) - .expect("2 cached <= 5 prompt is valid"); - right.record_content_token(); - right.record_tool_call_token(); - - let combined = left + right; - - assert_eq!(combined.prompt_tokens, 7); - assert_eq!(combined.cached_prompt_tokens, 3); - assert_eq!(combined.content_tokens, 2); - assert_eq!(combined.reasoning_tokens, 1); - assert_eq!(combined.tool_call_tokens, 2); - assert_eq!(combined.undeterminable_tokens, 1); - } - - #[test] - fn add_combines_image_and_audio_fields_too() { - let mut left = TokenUsage::new(); - left.record_input_image_tokens(3); - left.record_input_audio_tokens(7); - - let mut right = TokenUsage::new(); - right.record_input_image_tokens(4); - right.record_input_audio_tokens(1); - - let combined = left + right; - - assert_eq!(combined.input_image_tokens, 7); - assert_eq!(combined.input_audio_tokens, 8); + fn add_combines_every_field() { + assert_eq!(LEFT_USAGE + RIGHT_USAGE, COMBINED_USAGE); } #[test] @@ -398,17 +393,10 @@ mod tests { } #[test] - fn add_ref_combines_field_by_field() { - let mut left = TokenUsage::new(); - left.record_prompt_tokens(2); - - let mut right = TokenUsage::new(); - right.record_prompt_tokens(5); - let right_ref = &right; - - let combined = left + right_ref; + fn add_ref_combines_every_field() { + let right_usage = &RIGHT_USAGE; - assert_eq!(combined.prompt_tokens, 7); + assert_eq!(LEFT_USAGE + right_usage, COMBINED_USAGE); } #[test] @@ -442,7 +430,7 @@ mod tests { #[test] fn sum_over_empty_iter_returns_default() { - let summed: TokenUsage = std::iter::empty::().sum(); + let summed: TokenUsage = iter::empty::().sum(); assert_eq!(summed, TokenUsage::default()); } diff --git a/llama-cpp-bindings-types/src/tool_call_args_shape.rs b/llama-cpp-bindings-types/src/tool_call_args_shape.rs deleted file mode 100644 index 10f3b1fb4..000000000 --- a/llama-cpp-bindings-types/src/tool_call_args_shape.rs +++ /dev/null @@ -1,14 +0,0 @@ -use crate::bracketed_json_shape::BracketedJsonShape; -use crate::json_object_shape::JsonObjectShape; -use crate::key_value_xml_tags_shape::KeyValueXmlTagsShape; -use crate::paired_quote_shape::PairedQuoteShape; -use crate::xml_tags_shape::XmlTagsShape; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum ToolCallArgsShape { - BracketedJson(BracketedJsonShape), - JsonObject(JsonObjectShape), - KeyValueXmlTags(KeyValueXmlTagsShape), - PairedQuote(PairedQuoteShape), - XmlTags(XmlTagsShape), -} diff --git a/llama-cpp-bindings-types/src/tool_call_arguments.rs b/llama-cpp-bindings-types/src/tool_call_arguments.rs index 587ec130e..b00ffaf92 100644 --- a/llama-cpp-bindings-types/src/tool_call_arguments.rs +++ b/llama-cpp-bindings-types/src/tool_call_arguments.rs @@ -2,16 +2,27 @@ use serde::Deserialize; use serde::Serialize; use serde_json::Value; +use crate::invalid_arguments_json::InvalidArgumentsJson; + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum ToolCallArguments { ValidJson(Value), - InvalidJson(String), + InvalidJson { + raw: String, + error: InvalidArgumentsJson, + }, } impl ToolCallArguments { #[must_use] pub fn from_string(raw: String) -> Self { - serde_json::from_str::(&raw).map_or_else(|_| Self::InvalidJson(raw), Self::ValidJson) + match serde_json::from_str::(&raw) { + Ok(value) => Self::ValidJson(value), + Err(error) => Self::InvalidJson { + raw, + error: error.into(), + }, + } } } @@ -20,6 +31,7 @@ mod tests { use serde_json::json; use super::ToolCallArguments; + use crate::invalid_arguments_json::InvalidArgumentsJson; #[test] fn from_string_object_returns_valid() { @@ -46,17 +58,34 @@ mod tests { } #[test] - fn from_string_unparseable_returns_invalid() { + fn unparseable_arguments_keep_their_text_and_where_parsing_failed() { let raw = "{not really json".to_owned(); - let result = ToolCallArguments::from_string(raw.clone()); - assert_eq!(result, ToolCallArguments::InvalidJson(raw)); + assert_eq!( + ToolCallArguments::from_string(raw.clone()), + ToolCallArguments::InvalidJson { + raw, + error: InvalidArgumentsJson { + message: "key must be a string at line 1 column 2".to_owned(), + line: 1, + column: 2, + }, + } + ); } #[test] - fn from_string_empty_returns_invalid() { - let result = ToolCallArguments::from_string(String::new()); - - assert_eq!(result, ToolCallArguments::InvalidJson(String::new())); + fn empty_arguments_are_invalid_at_their_end() { + assert_eq!( + ToolCallArguments::from_string(String::new()), + ToolCallArguments::InvalidJson { + raw: String::new(), + error: InvalidArgumentsJson { + message: "EOF while parsing a value at line 1 column 0".to_owned(), + line: 1, + column: 0, + }, + } + ); } } diff --git a/llama-cpp-bindings-types/src/tool_call_markers.rs b/llama-cpp-bindings-types/src/tool_call_markers.rs deleted file mode 100644 index 1f6610cd3..000000000 --- a/llama-cpp-bindings-types/src/tool_call_markers.rs +++ /dev/null @@ -1,8 +0,0 @@ -use crate::tool_call_args_shape::ToolCallArgsShape; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ToolCallMarkers { - pub open: String, - pub close: String, - pub args_shape: ToolCallArgsShape, -} diff --git a/llama-cpp-bindings-types/src/tool_call_value_quote.rs b/llama-cpp-bindings-types/src/tool_call_value_quote.rs deleted file mode 100644 index aca34cbfa..000000000 --- a/llama-cpp-bindings-types/src/tool_call_value_quote.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ToolCallValueQuote { - pub open: String, - pub close: String, -} diff --git a/llama-cpp-bindings-types/src/xml_tags_shape.rs b/llama-cpp-bindings-types/src/xml_tags_shape.rs deleted file mode 100644 index c09634bea..000000000 --- a/llama-cpp-bindings-types/src/xml_tags_shape.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct XmlTagsShape { - pub function_open_prefix: String, - pub function_close: String, - pub parameter_open_prefix: String, - pub parameter_close: String, -} diff --git a/llama-cpp-bindings/Cargo.toml b/llama-cpp-bindings/Cargo.toml index 91d4eeb11..36d42e8eb 100644 --- a/llama-cpp-bindings/Cargo.toml +++ b/llama-cpp-bindings/Cargo.toml @@ -17,15 +17,14 @@ llama-cpp-gbnf = { workspace = true } llama-cpp-log-decoder = { workspace = true } llguidance = { workspace = true } log = { workspace = true } -nom = { workspace = true } once_cell = { workspace = true } -serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } toktrie = { workspace = true } [dev-dependencies] anyhow = { workspace = true } +llama-cpp-fault-injection = { workspace = true } llama-cpp-wrapper-error-fixture = { workspace = true } serial_test = { workspace = true } @@ -44,7 +43,7 @@ android-static-stdcxx = ["llama-cpp-bindings-sys/static-stdcxx"] system-ggml = ["llama-cpp-bindings-sys/system-ggml"] system-ggml-static = ["system-ggml", "llama-cpp-bindings-sys/system-ggml-static"] -[target.'cfg(all(target_os = "macos", any(target_arch = "aarch64", target_arch = "arm64")))'.dependencies] +[target.'cfg(all(target_os = "macos", target_arch = "aarch64"))'.dependencies] llama-cpp-bindings-sys = { workspace = true, features = ["metal"] } [lints] diff --git a/llama-cpp-bindings/build.rs b/llama-cpp-bindings/build.rs deleted file mode 100644 index fa6334b55..000000000 --- a/llama-cpp-bindings/build.rs +++ /dev/null @@ -1,5 +0,0 @@ -fn main() { - if let Ok(dir) = std::env::var("DEP_LLAMA_BACKENDS_DIR") { - println!("cargo:rustc-env=GGML_BACKENDS_DIR={dir}"); - } -} diff --git a/llama-cpp-bindings/src/backend_device_index.rs b/llama-cpp-bindings/src/backend_device_index.rs new file mode 100644 index 000000000..b88083e3d --- /dev/null +++ b/llama-cpp-bindings/src/backend_device_index.rs @@ -0,0 +1,12 @@ +/// The index of `device` among the backend devices ggml has registered. +#[must_use] +pub fn backend_device_index(device: llama_cpp_bindings_sys::ggml_backend_dev_t) -> Option { + let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; + + (0..device_count).position(|device_index| { + let registered_device = + unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; + + registered_device == device + }) +} diff --git a/llama-cpp-bindings/src/bare_json_tool_calls.rs b/llama-cpp-bindings/src/bare_json_tool_calls.rs deleted file mode 100644 index 853c839e6..000000000 --- a/llama-cpp-bindings/src/bare_json_tool_calls.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum BareJsonToolCalls { - Detect, - Ignore, -} diff --git a/llama-cpp-bindings/src/batch_add_error.rs b/llama-cpp-bindings/src/batch_add_error.rs index 146743362..d705ca403 100644 --- a/llama-cpp-bindings/src/batch_add_error.rs +++ b/llama-cpp-bindings/src/batch_add_error.rs @@ -1,22 +1,16 @@ -use std::num::TryFromIntError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; -#[derive(thiserror::Error, Debug, PartialEq, Eq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq, Eq)] pub enum BatchAddError { - #[error("Insufficient Space of {0}")] - InsufficientSpace(usize), - #[error("Empty buffer")] - EmptyBuffer, - #[error( - "a token in {sequence_ids} sequences does not fit a batch created for at most {n_seq_max} sequences per token" - )] - TooManySequenceIds { sequence_ids: i32, n_seq_max: i32 }, + #[error("the batch already holds the {n_batch} tokens one decode of the context takes")] + BatchFull { n_batch: u32 }, + #[error(transparent)] + TokenOutsideVocabulary(#[from] TokenOutsideVocabulary), + #[error("a token must belong to at least one sequence")] + NoSequenceIds, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), #[error("the batch already holds {n_tokens} tokens and one more would overflow i32")] TokenCountOverflow { n_tokens: i32 }, - #[error("{value_description} does not fit into {target_type}")] - IntegerOverflow { - value_description: &'static str, - target_type: &'static str, - #[source] - source: TryFromIntError, - }, } diff --git a/llama-cpp-bindings/src/chat_assistant_prefill.rs b/llama-cpp-bindings/src/chat_assistant_prefill.rs new file mode 100644 index 000000000..1c2fa42e1 --- /dev/null +++ b/llama-cpp-bindings/src/chat_assistant_prefill.rs @@ -0,0 +1,7 @@ +/// The beginning of an assistant answer the model continues: its reasoning, closed, and the +/// content written so far. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatAssistantPrefill { + pub reasoning_content: String, + pub content: String, +} diff --git a/llama-cpp-bindings/src/chat_assistant_turn.rs b/llama-cpp-bindings/src/chat_assistant_turn.rs new file mode 100644 index 000000000..8a7c6602f --- /dev/null +++ b/llama-cpp-bindings/src/chat_assistant_turn.rs @@ -0,0 +1,34 @@ +use crate::chat_assistant_prefill::ChatAssistantPrefill; + +/// How the rendered prompt leaves the assistant's turn, as llama.cpp's `add_generation_prompt` +/// and `continue_final_message` do. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatAssistantTurn { + /// The prompt ends after the last message. + Omitted, + /// The prompt opens a new assistant turn. + New, + /// The prompt opens an assistant turn whose reasoning starts with the given text and is left + /// open for the model to continue. + ContinueReasoning(String), + /// The prompt opens an assistant turn with the given reasoning closed and content left open + /// for the model to continue. + ContinueContent(ChatAssistantPrefill), +} + +impl From<&ChatAssistantTurn> for llama_cpp_bindings_sys::llama_rs_chat_assistant_turn { + fn from(assistant_turn: &ChatAssistantTurn) -> Self { + match assistant_turn { + ChatAssistantTurn::Omitted => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_ASSISTANT_TURN_OMITTED + } + ChatAssistantTurn::New => llama_cpp_bindings_sys::LLAMA_RS_CHAT_ASSISTANT_TURN_NEW, + ChatAssistantTurn::ContinueReasoning(_) => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_REASONING + } + ChatAssistantTurn::ContinueContent(_) => { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_ASSISTANT_TURN_CONTINUE_CONTENT + } + } + } +} diff --git a/llama-cpp-bindings/src/chat_classification.rs b/llama-cpp-bindings/src/chat_classification.rs new file mode 100644 index 000000000..b1cd0b028 --- /dev/null +++ b/llama-cpp-bindings/src/chat_classification.rs @@ -0,0 +1,54 @@ +use std::ops::Range; + +use crate::chat_output_completeness::ChatOutputCompleteness; +use crate::chat_piece_role::ChatPieceRole; +use crate::chat_spans::ChatSpans; +use crate::sampled_token_section::SampledTokenSection; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatClassification { + Spans(ChatSpans), + OutputDoesNotMatchFormat(ChatOutputCompleteness), +} + +impl ChatClassification { + /// Returns how the token whose text occupies `bytes` of the output is attributed, or `None` + /// while the classification cannot settle it yet. An answer that does not match the + /// template's format attributes no token to a section once it is finished. + #[must_use] + pub fn classify_token(&self, bytes: &Range) -> Option { + match self { + Self::Spans(spans) => spans.classify_token(bytes), + Self::OutputDoesNotMatchFormat(ChatOutputCompleteness::Partial) => None, + Self::OutputDoesNotMatchFormat(ChatOutputCompleteness::Finished) => { + Some(ChatPieceRole::Text(SampledTokenSection::Pending)) + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::chat_classification::ChatClassification; + use crate::chat_output_completeness::ChatOutputCompleteness; + use crate::chat_piece_role::ChatPieceRole; + use crate::sampled_token_section::SampledTokenSection; + + #[test] + fn a_partial_answer_that_does_not_match_the_format_settles_no_token() { + assert_eq!( + ChatClassification::OutputDoesNotMatchFormat(ChatOutputCompleteness::Partial) + .classify_token(&(0..3)), + None + ); + } + + #[test] + fn a_finished_answer_that_does_not_match_the_format_is_visible_text_of_no_section() { + assert_eq!( + ChatClassification::OutputDoesNotMatchFormat(ChatOutputCompleteness::Finished) + .classify_token(&(0..3)), + Some(ChatPieceRole::Text(SampledTokenSection::Pending)) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_grammar_trigger.rs b/llama-cpp-bindings/src/chat_grammar_trigger.rs new file mode 100644 index 000000000..e3c2a3d78 --- /dev/null +++ b/llama-cpp-bindings/src/chat_grammar_trigger.rs @@ -0,0 +1,90 @@ +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::token::LlamaToken; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatGrammarTrigger { + Token(LlamaToken), + Word(String), + Pattern(String), + PatternFull(String), +} + +impl ChatGrammarTrigger { + /// Builds the trigger the wrapper reports as `trigger_type` with `value`, reading its token + /// only for a token trigger. + /// + /// # Errors + /// + /// Returns [`llama_cpp_ffi_status::ffi_status_error::FfiStatusError`] for a trigger type unknown to this binding. + pub fn from_wrapper( + trigger_type: llama_cpp_bindings_sys::llama_rs_chat_grammar_trigger_type, + value: String, + token: impl FnOnce() -> LlamaToken, + ) -> Result { + match trigger_type { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN => { + Ok(Self::Token(token())) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD => { + Ok(Self::Word(value)) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN => { + Ok(Self::Pattern(value)) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL => { + Ok(Self::PatternFull(value)) + } + other => Err(FfiStatusError { + operation: "llama_rs_chat_session_grammar_trigger_type", + code: i64::from(other), + }), + } + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use crate::chat_grammar_trigger::ChatGrammarTrigger; + use crate::token::LlamaToken; + + fn trigger_of( + trigger_type: llama_cpp_bindings_sys::llama_rs_chat_grammar_trigger_type, + ) -> Result { + ChatGrammarTrigger::from_wrapper(trigger_type, "".to_owned(), || { + LlamaToken::new(151_657) + }) + } + + #[test] + fn every_wrapper_trigger_type_maps_to_its_trigger() { + assert_eq!( + [ + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_TOKEN, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_WORD, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, + ] + .map(trigger_of), + [ + Ok(ChatGrammarTrigger::Token(LlamaToken::new(151_657))), + Ok(ChatGrammarTrigger::Word("".to_owned())), + Ok(ChatGrammarTrigger::Pattern("".to_owned())), + Ok(ChatGrammarTrigger::PatternFull("".to_owned())), + ] + ); + } + + #[test] + fn an_unknown_wrapper_trigger_type_is_a_status_error() { + assert_eq!( + trigger_of(99), + Err(FfiStatusError { + operation: "llama_rs_chat_session_grammar_trigger_type", + code: 99, + }) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_message_parse_outcome.rs b/llama-cpp-bindings/src/chat_message_parse_outcome.rs index 833fbc596..10c277e2a 100644 --- a/llama-cpp-bindings/src/chat_message_parse_outcome.rs +++ b/llama-cpp-bindings/src/chat_message_parse_outcome.rs @@ -1,4 +1,4 @@ -use llama_cpp_bindings_types::ParsedChatMessage; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; use crate::raw_chat_message::RawChatMessage; @@ -8,51 +8,51 @@ pub enum ChatMessageParseOutcome { Unrecognized(RawChatMessage), } +impl ChatMessageParseOutcome { + /// The recognized message, or the raw answer the parser did not recognize. + /// + /// # Errors + /// + /// Returns the [`RawChatMessage`] when the answer does not match the template's format. + pub fn into_recognized(self) -> Result { + match self { + Self::Recognized(message) => Ok(message), + Self::Unrecognized(raw) => Err(raw), + } + } +} + #[cfg(test)] mod tests { - use llama_cpp_bindings_types::ParsedChatMessage; - use llama_cpp_bindings_types::ParsedToolCall; + use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; - use super::ChatMessageParseOutcome; + use crate::chat_message_parse_outcome::ChatMessageParseOutcome; + use crate::chat_output_completeness::ChatOutputCompleteness; use crate::raw_chat_message::RawChatMessage; - #[test] - fn both_variants_destructure_to_their_inner_payloads() { - let outcomes = [ - ChatMessageParseOutcome::Recognized(ParsedChatMessage::new( - "content".to_owned(), - "reasoning".to_owned(), - Vec::new(), - )), - ChatMessageParseOutcome::Unrecognized(RawChatMessage { - text: "raw input".to_owned(), - is_partial: false, - ffi_error_message: "parser bailed".to_owned(), - }), - ]; - - let mut saw_recognized = false; - let mut saw_unrecognized = false; - for outcome in outcomes { - match outcome { - ChatMessageParseOutcome::Recognized(parsed) => { - assert_eq!(parsed.content, "content"); - assert_eq!(parsed.reasoning_content, "reasoning"); - assert_eq!(parsed.tool_calls, Vec::::new()); - saw_recognized = true; - } - ChatMessageParseOutcome::Unrecognized(raw) => { - assert_eq!(raw.text, "raw input"); - assert!(!raw.is_partial); - assert_eq!(raw.ffi_error_message, "parser bailed"); - saw_unrecognized = true; - } - } + fn odd_answer() -> RawChatMessage { + RawChatMessage { + text: "Paris.".to_owned(), + completeness: ChatOutputCompleteness::Finished, + ffi_error_message: "the format differs".to_owned(), } + } - assert!( - saw_recognized && saw_unrecognized, - "both variants must dispatch through the match" + #[test] + fn a_recognized_answer_is_its_message() { + let message = ParsedChatMessage::new("Paris.".to_owned(), String::new(), Vec::new()); + + assert_eq!( + ChatMessageParseOutcome::Recognized(message.clone()).into_recognized(), + Ok(message) + ); + } + + #[test] + fn an_unrecognized_answer_is_its_raw_message() { + assert_eq!( + ChatMessageParseOutcome::Unrecognized(odd_answer()).into_recognized(), + Err(odd_answer()) ); } } diff --git a/llama-cpp-bindings/src/chat_message_parser.rs b/llama-cpp-bindings/src/chat_message_parser.rs deleted file mode 100644 index 81bb36439..000000000 --- a/llama-cpp-bindings/src/chat_message_parser.rs +++ /dev/null @@ -1,2633 +0,0 @@ -use std::ffi::CStr; -use std::ffi::CString; -use std::ffi::c_char; -use std::ptr; -use std::ptr::NonNull; - -use llama_cpp_bindings_types::ModelProvidedToolCallId; -use llama_cpp_bindings_types::ModelProvidedToolCallIdError; -use llama_cpp_bindings_types::ParsedChatMessage; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ReasoningMarkers; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_ffi_status::read_and_free_cpp_string; - -use crate::chat_message_parse_outcome::ChatMessageParseOutcome; -use crate::chat_template_tool_calls; -use crate::error::parse_chat_message_error::ParseChatMessageError; -use crate::model::LlamaModel; -use crate::raw_chat_message::RawChatMessage; -use crate::tool_call_format; -use crate::tool_call_format::ToolCallFormatOutcome; - -/// # Safety -/// -/// `free_error` must be the pointer populated by the preceding -/// `llama_rs_parsed_chat_free` call, or null. The destructor-threw arm reads and -/// frees it. -unsafe fn parsed_chat_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_free_status, - free_error: *mut c_char, -) -> Result<(), ParseChatMessageError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - free_error, - "llama_rs_parsed_chat_free", - "reported a thrown C++ exception without an error message", - ) - }?; - - Err(ParseChatMessageError::DestructorFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_free", - code: i64::from(other), - } - .into()), - } -} - -/// # Safety -/// -/// `out_error` must reference the pointer populated by a `llama_rs_parse_chat_message` call -/// that reported a thrown C++ exception. The error is read, freed, and the referenced pointer -/// is nulled so the later free in the caller does not double-free. -unsafe fn thrown_parse_exception_error( - out_error: *mut *mut c_char, - error_from_message: fn(String) -> ParseChatMessageError, -) -> ParseChatMessageError { - match unsafe { - read_and_free_cpp_string( - *out_error, - "llama_rs_parse_chat_message", - "reported a thrown C++ exception without an error message", - ) - } { - Ok(message) => { - unsafe { *out_error = ptr::null_mut() }; - - error_from_message(message) - } - Err(missing_message) => missing_message.into(), - } -} - -/// # Safety -/// -/// `handle` must be the parsed-chat handle (or null) and `out_error` must reference the -/// pointer populated by the preceding `llama_rs_parse_chat_message` call. In the CXX-exception -/// arms the error is read, freed, and the referenced pointer is nulled so the later free in the -/// caller does not double-free. -unsafe fn parse_chat_message_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parse_chat_message_status, - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, - out_error: *mut *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_OK => { - if handle.is_null() { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "success status contained a null parsed-chat handle", - } - .into()) - } else { - collect_parsed_chat_message(handle) - } - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - Err(unsafe { - thrown_parse_exception_error(out_error, |message| { - ParseChatMessageError::MessageUnrecognized { message } - }) - }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null tools parser argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null input argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_HANDLE_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null out_handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_parse_chat_message", - code: i64::from(other), - } - .into()), - } -} - -fn outcome_from_via_ffi_result( - via_ffi_result: Result, - input: &str, - is_partial: bool, -) -> Result { - match via_ffi_result { - Ok(parsed) => Ok(ChatMessageParseOutcome::Recognized(parsed)), - Err(ParseChatMessageError::MessageUnrecognized { message }) => { - Ok(ChatMessageParseOutcome::Unrecognized(RawChatMessage { - text: input.to_owned(), - is_partial, - ffi_error_message: message, - })) - } - Err(other) => Err(other), - } -} - -fn collect_parsed_chat_message( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, -) -> Result { - if handle.is_null() { - return Ok(ParsedChatMessage::default()); - } - - let content = read_parsed_chat_content(handle)?; - let reasoning_content = read_parsed_chat_reasoning_content(handle)?; - let count = read_parsed_chat_tool_call_count(handle)?; - - let mut tool_calls = Vec::with_capacity(count); - for index in 0..count { - let id = tool_call_id_reported_by_llama_cpp(read_parsed_chat_tool_call_id(handle, index)?); - let name = read_parsed_chat_tool_call_name(handle, index)?; - let arguments_json = read_parsed_chat_tool_call_arguments(handle, index)?; - - let arguments = ToolCallArguments::from_string(arguments_json); - tool_calls.push(ParsedToolCall::new(id, name, arguments)); - } - - Ok(ParsedChatMessage::new( - content, - reasoning_content, - tool_calls, - )) -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_content` call (or null when no value/error was produced); each is -/// read and freed in exactly one match arm. -unsafe fn parsed_chat_content_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_content_status, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_content") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_parsed_chat_content", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_content", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_content( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_content( - handle, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_content_status_to_result(status, out_string, out_error) } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_reasoning_content` call (or null when no value/error was produced); -/// each is read and freed in exactly one match arm. -unsafe fn parsed_chat_reasoning_content_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_reasoning_content_status, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_reasoning_content") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_reasoning_content", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_reasoning_content", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_reasoning_content( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_reasoning_content( - handle, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_reasoning_content_status_to_result(status, out_string, out_error) } -} - -/// # Safety -/// -/// `out_error` must be the pointer populated by the preceding -/// `llama_rs_parsed_chat_tool_call_count` call (or null when no error was produced); it is -/// freed in exactly one match arm. -unsafe fn parsed_chat_tool_call_count_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_count_status, - out_count: usize, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_OK => Ok(out_count), - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_tool_call_count", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_count", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_OUT_COUNT_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_count", - detail: "was given a null out_count argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_count", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_tool_call_count( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, -) -> Result { - let mut out_count: usize = 0; - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_count( - handle, - &raw mut out_count, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_tool_call_count_status_to_result(status, out_count, out_error) } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_tool_call_id` call (or null when no value/error was produced); each -/// is read and freed in exactly one match arm. -unsafe fn parsed_chat_tool_call_id_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_id_status, - index: usize, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_tool_call_id") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_INDEX_OUT_OF_BOUNDS => { - Err(ParseChatMessageError::ToolCallIdIndexOutOfBounds { index }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_tool_call_id", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_id", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_tool_call_id( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, - index: usize, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_id( - handle, - index, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_tool_call_id_status_to_result(status, index, out_string, out_error) } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_tool_call_name` call (or null when no value/error was produced); each -/// is read and freed in exactly one match arm. -unsafe fn parsed_chat_tool_call_name_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_name_status, - index: usize, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_tool_call_name") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_INDEX_OUT_OF_BOUNDS => { - Err(ParseChatMessageError::ToolCallNameIndexOutOfBounds { index }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_tool_call_name", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_name", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_tool_call_name( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, - index: usize, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_name( - handle, - index, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { parsed_chat_tool_call_name_status_to_result(status, index, out_string, out_error) } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_parsed_chat_tool_call_arguments` call (or null when no value/error was produced); -/// each is read and freed in exactly one match arm. -unsafe fn parsed_chat_tool_call_arguments_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_arguments_status, - index: usize, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_OK => { - consume_accessor_string(out_string, "llama_rs_parsed_chat_tool_call_arguments") - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_INDEX_OUT_OF_BOUNDS => { - Err(ParseChatMessageError::ToolCallArgumentsIndexOutOfBounds { index }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_OUT_OF_MEMORY => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = - unsafe { read_and_free_cpp_string(out_error, "llama_rs_parsed_chat_tool_call_arguments", "reported a thrown C++ exception without an error message") }?; - Err(ParseChatMessageError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_HANDLE_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "was given a null handle argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_OUT_STRING_ARG => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "was given a null out_string argument", - } - .into()) - } - other => { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_string) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - code: i64::from(other), - } - .into()) - } - } -} - -fn read_parsed_chat_tool_call_arguments( - handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, - index: usize, -) -> Result { - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_arguments( - handle, - index, - &raw mut out_string, - &raw mut out_error, - ) - }; - unsafe { - parsed_chat_tool_call_arguments_status_to_result(status, index, out_string, out_error) - } -} - -fn consume_accessor_string( - ptr: *mut c_char, - operation: &'static str, -) -> Result { - if ptr.is_null() { - return Err(crate::FfiContractError { - operation, - detail: "success status contained a null string", - } - .into()); - } - let bytes = unsafe { CStr::from_ptr(ptr) }.to_bytes().to_vec(); - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(ptr) }; - Ok(String::from_utf8(bytes)?) -} - -struct ReasoningSplit { - reasoning: String, - content: String, -} - -fn restore_partial_reasoning( - parsed: &mut ParsedChatMessage, - input: &str, - reasoning_markers: Option<&ReasoningMarkers>, - is_partial: bool, -) { - if !is_partial { - return; - } - if reasoning_markers.is_some_and(|markers| input.contains(&markers.open)) { - let split = split_reasoning_prefix(input, reasoning_markers, None, true); - parsed.reasoning_content = split.reasoning; - parsed.content = split.content; - return; - } - if let Some(open) = reasoning_markers.map(|markers| markers.open.trim()) - && let Some(reasoning) = parsed.reasoning_content.trim_start().strip_prefix(open) - { - parsed.reasoning_content = reasoning.to_owned(); - } -} - -fn split_reasoning_prefix( - input: &str, - reasoning_markers: Option<&ReasoningMarkers>, - tool_call_open: Option<&str>, - is_partial: bool, -) -> ReasoningSplit { - let content_only = || ReasoningSplit { - reasoning: String::new(), - content: prefix_before_optional(input, tool_call_open), - }; - - let Some(reasoning_markers) = reasoning_markers else { - return content_only(); - }; - let Some(open_pos) = input.find(&reasoning_markers.open) else { - return content_only(); - }; - - let after_open = &input[open_pos + reasoning_markers.open.len()..]; - let closing_marker = reasoning_markers - .closes - .iter() - .enumerate() - .filter_map(|(marker_index, marker)| { - after_open - .find(marker) - .map(|offset| (offset, marker_index, marker)) - }) - .min_by_key(|(offset, marker_index, _)| (*offset, *marker_index)); - let Some((close_offset, _, close_marker)) = closing_marker else { - return if is_partial { - ReasoningSplit { - reasoning: prefix_before_optional(after_open, tool_call_open), - content: input[..open_pos].to_owned(), - } - } else { - content_only() - }; - }; - - let reasoning = after_open[..close_offset].to_owned(); - let after_close = &after_open[close_offset + close_marker.len()..]; - - ReasoningSplit { - reasoning, - content: prefix_before_optional(after_close, tool_call_open), - } -} - -fn prefix_before_optional(text: &str, marker: Option<&str>) -> String { - marker.map_or_else( - || text.to_owned(), - |marker| { - text.find(marker) - .map_or_else(|| text.to_owned(), |pos| text[..pos].to_owned()) - }, - ) -} - -fn tool_call_id_reported_by_llama_cpp(llama_cpp_tool_call_id: String) -> ParsedToolCallId { - match ModelProvidedToolCallId::try_from(llama_cpp_tool_call_id) { - Ok(model_provided_tool_call_id) => { - ParsedToolCallId::ModelProvided(model_provided_tool_call_id) - } - Err(ModelProvidedToolCallIdError::Empty) => ParsedToolCallId::NotProvidedByModel, - } -} - -/// # Safety -/// -/// `out_error` must reference the pointer populated by the preceding -/// `llama_rs_chat_tools_parser_create` call (or null); it is read, freed, and nulled only in -/// the CXX-exception arm. `tools_parser` must be the pointer populated by the same call. -unsafe fn chat_tools_parser_create_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_tools_parser_create_status, - tools_parser: *mut llama_cpp_bindings_sys::llama_rs_chat_tools_parser, - out_error: *mut *mut c_char, -) -> Result, ParseChatMessageError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK => NonNull::new(tools_parser) - .ok_or_else(|| { - crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "success status contained a null tools parser handle", - } - .into() - }), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY => { - Err(ParseChatMessageError::ToolsNotAnArray) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - *out_error, - "llama_rs_chat_tools_parser_create", - "reported a thrown C++ exception without an error message", - ) - }?; - - unsafe { *out_error = ptr::null_mut() }; - - Err(ParseChatMessageError::ToolsParserBuildFailed { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "was given a null parser argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "was given a null tools_json argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "was given a null out_tools_parser argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_tools_parser_create", - code: i64::from(other), - } - .into()), - } -} - -/// # Safety -/// -/// `out_error` must be the pointer populated by the preceding -/// `llama_rs_chat_tools_parser_free` call, or null. The destructor-threw arm reads and frees it. -unsafe fn chat_tools_parser_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_tools_parser_free_status, - out_error: *mut c_char, -) -> Result<(), ParseChatMessageError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_chat_tools_parser_free", - "reported a thrown C++ exception without an error message", - ) - }?; - - Err(ParseChatMessageError::DestructorFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_tools_parser_free", - code: i64::from(other), - } - .into()), - } -} - -/// Parses generated chat messages against one set of tools, building the -/// model-specific tools parser once for every message it parses. -pub struct ChatMessageParser { - reasoning_markers: Option, - tools_parser: NonNull, -} - -unsafe impl Send for ChatMessageParser {} - -impl ChatMessageParser { - /// # Errors - /// - /// Returns [`ParseChatMessageError`] when reasoning-marker detection fails, the model has - /// no chat parser, `tools_json` contains a NUL byte or is not a JSON array, or the tools - /// parser cannot be built. - pub fn new(model: &LlamaModel, tools_json: &str) -> Result { - let reasoning_markers = model.reasoning_markers()?.cloned(); - let chat_parser = model.chat_parser_ptr()?; - let tools_json_cstring = - CString::new(tools_json).map_err(ParseChatMessageError::ToolsContainNulByte)?; - let mut out_tools_parser: *mut llama_cpp_bindings_sys::llama_rs_chat_tools_parser = - ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_tools_parser_create( - chat_parser, - tools_json_cstring.as_ptr(), - &raw mut out_tools_parser, - &raw mut out_error, - ) - }; - - let tools_parser = unsafe { - chat_tools_parser_create_status_to_result(status, out_tools_parser, &raw mut out_error) - }; - - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - - Ok(Self { - reasoning_markers, - tools_parser: tools_parser?, - }) - } - - /// # Errors - /// - /// Returns [`ParseChatMessageError`] when `input` contains a NUL byte, the FFI returns a - /// non-OK status other than a message parse exception, or accessor strings are not valid - /// UTF-8. - pub fn parse( - &self, - input: &str, - is_partial: bool, - ) -> Result { - let reasoning_markers = self.reasoning_markers.as_ref(); - - for candidate in chat_template_tool_calls::known_marker_candidates() { - match tool_call_format::try_parse(input, &candidate) { - ToolCallFormatOutcome::NoMatch => {} - ToolCallFormatOutcome::Parsed(calls) => { - let split = split_reasoning_prefix( - input, - reasoning_markers, - Some(&candidate.open), - is_partial, - ); - return Ok(ChatMessageParseOutcome::Recognized(ParsedChatMessage::new( - split.content, - split.reasoning, - calls, - ))); - } - ToolCallFormatOutcome::Failed(_shape_does_not_fit) => {} - } - } - - let via_ffi_result = self.parse_via_ffi(input, is_partial).map(|mut parsed| { - restore_partial_reasoning(&mut parsed, input, reasoning_markers, is_partial); - parsed - }); - - outcome_from_via_ffi_result(via_ffi_result, input, is_partial) - } - - fn parse_via_ffi( - &self, - input: &str, - is_partial: bool, - ) -> Result { - let input_cstring = - CString::new(input).map_err(ParseChatMessageError::InputContainsNulByte)?; - - let mut handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_parse_chat_message( - self.tools_parser.as_ptr(), - input_cstring.as_ptr(), - i32::from(is_partial), - &raw mut handle, - &raw mut out_error, - ) - }; - - let parsed = - unsafe { parse_chat_message_status_to_result(status, handle, &raw mut out_error) }; - - let mut free_error: *mut c_char = ptr::null_mut(); - let free_status = unsafe { - llama_cpp_bindings_sys::llama_rs_parsed_chat_free(handle, &raw mut free_error) - }; - let freed = unsafe { parsed_chat_free_status_to_result(free_status, free_error) }; - - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - - match parsed { - Ok(message) => freed.map(|()| message), - Err(parse_failure) => { - if let Err(destructor_failure) = freed { - log::error!("{destructor_failure}"); - } - - Err(parse_failure) - } - } - } -} - -impl Drop for ChatMessageParser { - fn drop(&mut self) { - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_tools_parser_free( - self.tools_parser.as_ptr(), - &raw mut out_error, - ) - }; - - if let Err(destructor_failure) = - unsafe { chat_tools_parser_free_status_to_result(status, out_error) } - { - log::error!("{destructor_failure}"); - } - } -} - -#[cfg(test)] -mod tests { - use std::ffi::CStr; - use std::ffi::c_char; - use std::mem::discriminant; - use std::ptr; - - use llama_cpp_bindings_types::ModelProvidedToolCallId; - use llama_cpp_bindings_types::ParsedChatMessage; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ParsedToolCallId; - use llama_cpp_bindings_types::ReasoningMarkers; - use llama_cpp_bindings_types::ToolCallArguments; - use serde_json::json; - - use super::ReasoningSplit; - use super::chat_tools_parser_create_status_to_result; - use super::chat_tools_parser_free_status_to_result; - use super::outcome_from_via_ffi_result; - use super::parse_chat_message_status_to_result; - use super::parsed_chat_content_status_to_result; - use super::parsed_chat_free_status_to_result; - use super::parsed_chat_reasoning_content_status_to_result; - use super::parsed_chat_tool_call_arguments_status_to_result; - use super::parsed_chat_tool_call_count_status_to_result; - use super::parsed_chat_tool_call_id_status_to_result; - use super::parsed_chat_tool_call_name_status_to_result; - use super::restore_partial_reasoning; - use super::split_reasoning_prefix; - use super::tool_call_id_reported_by_llama_cpp; - use crate::chat_message_parse_outcome::ChatMessageParseOutcome; - use crate::error::parse_chat_message_error::ParseChatMessageError; - use crate::raw_chat_message::RawChatMessage; - #[test] - fn parse_chat_message_success_with_null_handle_is_contract_error() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_OK, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "success status contained a null parsed-chat handle", - } - )) - )); - } - - #[test] - fn parse_chat_message_allocation_failed_is_not_enough_memory() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parse_chat_message_cxx_exception_is_message_unrecognized_and_nulls_error() { - let mut out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the message could not be parsed".as_ptr()) - }; - let result = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::MessageUnrecognized { - message: "the message could not be parsed".to_owned() - }) - ); - - assert!( - out_error.is_null(), - "the reclaimed pointer must be nulled so the caller does not free it twice" - ); - } - - #[test] - fn parse_chat_message_cxx_exception_without_an_error_message_is_a_contract_error() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - result.unwrap_err(), - ParseChatMessageError::FfiContract(crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "reported a thrown C++ exception without an error message", - }) - ); - } - - #[test] - fn parse_chat_message_unknown_status_is_preserved() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - parse_chat_message_status_to_result(255, ptr::null_mut(), &raw mut out_error) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parse_chat_message", - code: 255, - })) - ); - } - - #[test] - fn parsed_chat_content_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_OK, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_content_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_content_cxx_exception_is_reported() { - let out_error = - unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"content read failed".as_ptr()) }; - let result = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "content read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_content_unknown_status_is_preserved() { - let result = - unsafe { parsed_chat_content_status_to_result(255, ptr::null_mut(), ptr::null_mut()) }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_content", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_reasoning_content_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_OK, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_reasoning_content_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_reasoning_content_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"reasoning read failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "reasoning read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_reasoning_content_unknown_status_is_preserved() { - let result = unsafe { - parsed_chat_reasoning_content_status_to_result(255, ptr::null_mut(), ptr::null_mut()) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_reasoning_content", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_tool_call_count_ok_returns_count() { - let result = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_OK, - 7, - ptr::null_mut(), - ) - }; - - assert_eq!(result.unwrap(), 7); - } - - #[test] - fn parsed_chat_tool_call_count_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_count_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"tool-call count failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "tool-call count failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_tool_call_count_unknown_status_is_preserved() { - let result = - unsafe { parsed_chat_tool_call_count_status_to_result(255, 0, ptr::null_mut()) }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_count", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_tool_call_id_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_OK, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_tool_call_id_out_of_bounds_carries_index() { - let result = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_INDEX_OUT_OF_BOUNDS, - 4, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ToolCallIdIndexOutOfBounds { index: 4 }) - ); - } - - #[test] - fn parsed_chat_tool_call_id_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_id_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"tool-call id read failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "tool-call id read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_tool_call_id_unknown_status_is_preserved() { - let result = unsafe { - parsed_chat_tool_call_id_status_to_result(255, 0, ptr::null_mut(), ptr::null_mut()) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_id", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_tool_call_name_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_OK, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_tool_call_name_out_of_bounds_carries_index() { - let result = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_INDEX_OUT_OF_BOUNDS, - 2, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ToolCallNameIndexOutOfBounds { index: 2 }) - ); - } - - #[test] - fn parsed_chat_tool_call_name_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_name_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"tool-call name read failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "tool-call name read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_tool_call_name_unknown_status_is_preserved() { - let result = unsafe { - parsed_chat_tool_call_name_status_to_result(255, 0, ptr::null_mut(), ptr::null_mut()) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_name", - code: 255, - })) - )); - } - - #[test] - fn parsed_chat_tool_call_arguments_success_with_null_string_is_contract_error() { - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_OK, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "success status contained a null string", - } - )) - )); - } - - #[test] - fn parsed_chat_tool_call_arguments_out_of_bounds_carries_index() { - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_INDEX_OUT_OF_BOUNDS, - 9, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ToolCallArgumentsIndexOutOfBounds { index: 9 }) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_cxx_exception_is_reported() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"tool-call arguments read failed".as_ptr()) - }; - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::Reported { - message: "tool-call arguments read failed".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_unknown_status_is_preserved() { - let result = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - 255, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert!(matches!( - result, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - code: 255, - })) - )); - } - - #[test] - fn split_reasoning_prefix_without_markers_returns_content_up_to_tool_call_open() { - let ReasoningSplit { reasoning, content } = - split_reasoning_prefix("answerrest", None, Some(""), false); - - assert_eq!(reasoning, ""); - assert_eq!(content, "answer"); - } - - #[test] - fn split_reasoning_prefix_with_missing_open_marker_returns_content_only() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = - split_reasoning_prefix("plain answer", Some(&markers), Some(""), false); - - assert_eq!(reasoning, ""); - assert_eq!(content, "plain answer"); - } - - #[test] - fn split_reasoning_prefix_with_missing_close_marker_returns_content_only() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = - split_reasoning_prefix("unterminated", Some(&markers), Some(""), false); - - assert_eq!(reasoning, ""); - assert_eq!(content, "unterminated"); - } - - #[test] - fn split_reasoning_prefix_with_partial_unclosed_marker_returns_reasoning() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = split_reasoning_prefix( - "prefixunfinishedtail", - Some(&markers), - Some(""), - true, - ); - - assert_eq!(reasoning, "unfinished"); - assert_eq!(content, "prefix"); - } - - #[test] - fn split_reasoning_prefix_without_tool_marker_preserves_all_partial_reasoning() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = - split_reasoning_prefix("unfinished", Some(&markers), None, true); - - assert_eq!(reasoning, "unfinished"); - assert_eq!(content, ""); - } - - #[test] - fn split_reasoning_prefix_extracts_reasoning_and_trailing_content() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let ReasoningSplit { reasoning, content } = split_reasoning_prefix( - "deduceanswertail", - Some(&markers), - Some(""), - false, - ); - - assert_eq!(reasoning, "deduce"); - assert_eq!(content, "answer"); - } - - #[test] - fn restore_partial_reasoning_preserves_non_partial_parser_result() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = - ParsedChatMessage::new("parsed content".to_owned(), String::new(), Vec::new()); - - restore_partial_reasoning(&mut parsed, "unfinished", Some(&markers), false); - - assert_eq!(parsed.content, "parsed content"); - assert_eq!(parsed.reasoning_content, ""); - } - - #[test] - fn restore_partial_reasoning_preserves_existing_reasoning() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = ParsedChatMessage::new( - "parsed content".to_owned(), - "parsed reasoning".to_owned(), - Vec::new(), - ); - - restore_partial_reasoning(&mut parsed, "plain response", Some(&markers), true); - - assert_eq!(parsed.content, "parsed content"); - assert_eq!(parsed.reasoning_content, "parsed reasoning"); - } - - #[test] - fn restore_partial_reasoning_removes_open_marker_from_parser_result() { - let markers = ReasoningMarkers { - open: "\n[THINK]\n".to_owned(), - closes: vec!["[/THINK]".to_owned()], - }; - let mut parsed = ParsedChatMessage::new( - String::new(), - "[THINK]parsed reasoning".to_owned(), - Vec::new(), - ); - - restore_partial_reasoning(&mut parsed, "complete response", Some(&markers), true); - - assert_eq!(parsed.content, ""); - assert_eq!(parsed.reasoning_content, "parsed reasoning"); - } - - #[test] - fn restore_partial_reasoning_preserves_unclosed_reasoning_whitespace() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = - ParsedChatMessage::new(String::new(), "normalized reasoning".to_owned(), Vec::new()); - - restore_partial_reasoning(&mut parsed, "\n\nreasoning", Some(&markers), true); - - assert_eq!(parsed.content, ""); - assert_eq!(parsed.reasoning_content, "\n\nreasoning"); - } - - #[test] - fn restore_partial_reasoning_preserves_closed_reasoning_whitespace() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = ParsedChatMessage::new( - "answer".to_owned(), - "normalized reasoning".to_owned(), - Vec::new(), - ); - - restore_partial_reasoning( - &mut parsed, - "\n\nreasoninganswer", - Some(&markers), - true, - ); - - assert_eq!(parsed.content, "answer"); - assert_eq!(parsed.reasoning_content, "\n\nreasoning"); - } - - #[test] - fn restore_partial_reasoning_removes_open_marker_after_parser_whitespace() { - let markers = ReasoningMarkers { - open: "\n[THINK]\n".to_owned(), - closes: vec!["[/THINK]".to_owned()], - }; - let mut parsed = ParsedChatMessage::new( - String::new(), - "\n[THINK]parsed reasoning".to_owned(), - Vec::new(), - ); - - restore_partial_reasoning(&mut parsed, "complete response", Some(&markers), true); - - assert_eq!(parsed.content, ""); - assert_eq!(parsed.reasoning_content, "parsed reasoning"); - } - - #[test] - fn restore_partial_reasoning_preserves_result_without_open_marker() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = - ParsedChatMessage::new("parsed content".to_owned(), String::new(), Vec::new()); - - restore_partial_reasoning(&mut parsed, "unfinished", Some(&markers), true); - - assert_eq!(parsed.content, "parsed content"); - assert_eq!(parsed.reasoning_content, ""); - } - - #[test] - fn restore_partial_reasoning_recovers_unclosed_reasoning() { - let markers = ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned()], - }; - let mut parsed = - ParsedChatMessage::new("unfinished".to_owned(), String::new(), Vec::new()); - - restore_partial_reasoning(&mut parsed, "unfinished", Some(&markers), true); - - assert_eq!(parsed.content, ""); - assert_eq!(parsed.reasoning_content, "unfinished"); - } - - #[test] - fn outcome_from_via_ffi_result_recognized_keeps_a_tool_call_the_model_gave_no_id() { - let parsed = ParsedChatMessage::new( - "answer".to_owned(), - String::new(), - vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "tool".to_owned(), - ToolCallArguments::ValidJson(json!({})), - )], - ); - - let outcome = outcome_from_via_ffi_result(Ok(parsed.clone()), "answer", false); - - assert_eq!( - outcome.unwrap(), - ChatMessageParseOutcome::Recognized(parsed) - ); - } - - #[test] - fn an_empty_llama_cpp_tool_call_id_is_not_provided_by_the_model() { - assert_eq!( - tool_call_id_reported_by_llama_cpp(String::new()), - ParsedToolCallId::NotProvidedByModel - ); - } - - #[test] - fn a_non_empty_llama_cpp_tool_call_id_is_provided_by_the_model() { - assert_eq!( - Ok(tool_call_id_reported_by_llama_cpp("abc123def".to_owned())), - ModelProvidedToolCallId::try_from("abc123def".to_owned()) - .map(ParsedToolCallId::ModelProvided) - ); - } - - #[test] - fn outcome_from_via_ffi_result_message_unrecognized_is_unrecognized_with_raw_message() { - let outcome = outcome_from_via_ffi_result( - Err(ParseChatMessageError::MessageUnrecognized { - message: "boom".to_owned(), - }), - "garbled", - true, - ); - - assert_eq!( - outcome.unwrap(), - ChatMessageParseOutcome::Unrecognized(RawChatMessage { - text: "garbled".to_owned(), - is_partial: true, - ffi_error_message: "boom".to_owned(), - }) - ); - } - - #[test] - fn outcome_from_via_ffi_result_parser_creation_failure_propagates() { - let outcome = outcome_from_via_ffi_result( - Err(ParseChatMessageError::ParserCreationFailed { - message: "the parser could not be built".to_owned(), - }), - "garbled", - true, - ); - - assert_eq!( - discriminant(&outcome.unwrap_err()), - discriminant(&ParseChatMessageError::ParserCreationFailed { - message: String::new() - }) - ); - } - - #[test] - fn outcome_from_via_ffi_result_other_error_propagates() { - let outcome = outcome_from_via_ffi_result(Err(ParseChatMessageError::NoVocab), "x", false); - - assert_eq!( - discriminant(&outcome.unwrap_err()), - discriminant(&ParseChatMessageError::NoVocab) - ); - } - - #[test] - fn parsed_chat_free_ok_is_success() { - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_OK, - ptr::null_mut(), - ) - }; - - assert!( - result.is_ok(), - "a clean destructor must not report a failure" - ); - } - - #[test] - fn parsed_chat_free_allocation_failed_is_not_enough_memory() { - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ) - }; - - assert_eq!(result.err(), Some(ParseChatMessageError::NotEnoughMemory)); - } - - #[test] - fn parsed_chat_free_llama_cpp_out_of_memory_is_preserved() { - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_free_destructor_threw_surfaces_the_message() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the destructor threw".as_ptr()) - }; - let result = unsafe { - parsed_chat_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::DestructorFailed { - message: "the destructor threw".to_owned() - }) - ); - } - - #[test] - fn parsed_chat_free_unknown_status_is_preserved() { - let result = unsafe { parsed_chat_free_status_to_result(255, ptr::null_mut()) }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_parsed_chat_free", - code: 255, - })) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_null_tools_parser_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null tools parser argument", - } - )) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_null_input_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null input argument", - } - )) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_null_out_handle_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_HANDLE_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null out_handle argument", - } - )) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_null_out_error_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parse_chat_message", - detail: "was given a null out_error argument", - } - )) - ); - } - - #[test] - fn parse_chat_message_status_to_result_maps_llama_cpp_out_of_memory() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - parse_chat_message_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_content_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_NULL_HANDLE_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_content_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_NULL_OUT_STRING_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_content", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_content_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_reasoning_content_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_HANDLE_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_reasoning_content_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_reasoning_content_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_NULL_OUT_STRING_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_reasoning_content", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_reasoning_content_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_reasoning_content_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_REASONING_CONTENT_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_count_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_HANDLE_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_count", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_count_status_to_result_maps_null_out_count_arg() { - let outcome = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_NULL_OUT_COUNT_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_count", - detail: "was given a null out_count argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_count_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_tool_call_count_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_COUNT_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_id_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_HANDLE_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_id_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_NULL_OUT_STRING_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_id", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_id_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_tool_call_id_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ID_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_name_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_HANDLE_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_name_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_NULL_OUT_STRING_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_name", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_name_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_tool_call_name_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_NAME_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_status_to_result_maps_null_handle_arg() { - let outcome = unsafe { - parsed_chat_tool_call_arguments_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_HANDLE_ARG, - 0, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "was given a null handle argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_status_to_result_maps_null_out_string_arg() { - let outcome = unsafe { - parsed_chat_tool_call_arguments_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_NULL_OUT_STRING_ARG, 0, ptr::null_mut(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_parsed_chat_tool_call_arguments", - detail: "was given a null out_string argument", - } - )) - ); - } - - #[test] - fn parsed_chat_tool_call_arguments_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - parsed_chat_tool_call_arguments_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_PARSED_CHAT_TOOL_CALL_ARGUMENTS_LLAMA_CPP_OUT_OF_MEMORY, 0, ptr::null_mut(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - fn tools_parser_creation_error( - status: llama_cpp_bindings_sys::llama_rs_chat_tools_parser_create_status, - message: Option<&CStr>, - ) -> ParseChatMessageError { - let mut out_error = message.map_or(ptr::null_mut(), |message| unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(message.as_ptr()) - }); - - let error = unsafe { - chat_tools_parser_create_status_to_result(status, ptr::null_mut(), &raw mut out_error) - } - .expect_err("the status must map to an error"); - - assert!(out_error.is_null(), "a reported message must be reclaimed"); - - error - } - - #[test] - fn tools_parser_creation_reports_a_success_without_a_parser_as_a_contract_error() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK, - None - ), - ParseChatMessageError::FfiContract(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "success status contained a null tools parser handle", - }) - ); - } - - #[test] - fn tools_parser_creation_reports_tools_that_are_not_an_array() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY, - None - ), - ParseChatMessageError::ToolsNotAnArray - ); - } - - #[test] - fn tools_parser_creation_reports_running_out_of_memory() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED, - None - ), - ParseChatMessageError::NotEnoughMemory - ); - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - None - ), - ParseChatMessageError::LlamaCppOutOfMemory - ); - } - - #[test] - fn tools_parser_creation_reports_the_build_failure_message() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - Some(c"tools do not fit the template") - ), - ParseChatMessageError::ToolsParserBuildFailed { - message: "tools do not fit the template".to_owned() - } - ); - } - - #[test] - fn tools_parser_creation_reports_a_build_failure_without_a_message_as_a_contract_error() { - assert_eq!( - tools_parser_creation_error( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - None - ), - ParseChatMessageError::FfiContract(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail: "reported a thrown C++ exception without an error message", - }) - ); - } - - #[test] - fn tools_parser_creation_reports_every_null_argument_as_a_contract_error() { - for (status, detail) in [ - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG, - "was given a null parser argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG, - "was given a null tools_json argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG, - "was given a null out_tools_parser argument", - ), - ( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG, - "was given a null out_error argument", - ), - ] { - assert_eq!( - tools_parser_creation_error(status, None), - ParseChatMessageError::FfiContract(crate::FfiContractError { - operation: "llama_rs_chat_tools_parser_create", - detail, - }) - ); - } - } - - #[test] - fn tools_parser_creation_preserves_an_unknown_status() { - assert_eq!( - tools_parser_creation_error(255, None), - ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_tools_parser_create", - code: 255, - }) - ); - } - - #[test] - fn tools_parser_release_maps_every_status() { - assert_eq!( - unsafe { - chat_tools_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK, - ptr::null_mut(), - ) - }, - Ok(()) - ); - assert_eq!( - unsafe { - chat_tools_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ) - }, - Err(ParseChatMessageError::NotEnoughMemory) - ); - assert_eq!( - unsafe { - chat_tools_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ) - }, - Err(ParseChatMessageError::LlamaCppOutOfMemory) - ); - assert_eq!( - unsafe { - chat_tools_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - llama_cpp_bindings_sys::llama_rs_string_dup(c"destructor failed".as_ptr()), - ) - }, - Err(ParseChatMessageError::DestructorFailed { - message: "destructor failed".to_owned() - }) - ); - assert_eq!( - unsafe { chat_tools_parser_free_status_to_result(255, ptr::null_mut()) }, - Err(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_tools_parser_free", - code: 255, - })) - ); - } -} diff --git a/llama-cpp-bindings/src/chat_output.rs b/llama-cpp-bindings/src/chat_output.rs new file mode 100644 index 000000000..60ddaba2f --- /dev/null +++ b/llama-cpp-bindings/src/chat_output.rs @@ -0,0 +1,85 @@ +use std::ops::Range; + +use llama_cpp_bindings_sys::LLAMA_TOKEN_NULL; +use llama_cpp_bindings_sys::llama_token; + +use crate::token::LlamaToken; + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ChatOutput { + text: String, + byte_tokens: Vec, +} + +impl From<&str> for ChatOutput { + /// Holds `text` without tokens, the way `common_chat_input` holds plain text. + fn from(text: &str) -> Self { + Self { + text: text.to_owned(), + byte_tokens: vec![LLAMA_TOKEN_NULL; text.len()], + } + } +} + +impl ChatOutput { + /// Appends the text `token` decoded to, recording the token at the first byte of its piece + /// the way `common_chat_input` lays tokens over the text, and returns the bytes the piece + /// occupies. + pub fn push(&mut self, token: LlamaToken, piece: &str) -> Range { + let start = self.text.len(); + + self.text.push_str(piece); + self.byte_tokens.extend((0..piece.len()).map(|offset| { + if offset == 0 { + token.0 + } else { + LLAMA_TOKEN_NULL + } + })); + + start..self.text.len() + } + + #[must_use] + pub fn text(&self) -> &str { + &self.text + } + + #[must_use] + pub fn byte_tokens(&self) -> &[llama_token] { + &self.byte_tokens + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_bindings_sys::LLAMA_TOKEN_NULL; + + use crate::chat_output::ChatOutput; + use crate::token::LlamaToken; + + #[test] + fn each_token_owns_the_bytes_of_its_piece_and_marks_its_first_byte() { + let mut output = ChatOutput::default(); + + let pieces = [ + output.push(LlamaToken::new(7), ""), + output.push(LlamaToken::new(9), ""), + output.push(LlamaToken::new(11), "hi"), + ]; + + assert_eq!(pieces, [0..3, 3..3, 3..5]); + assert_eq!(output.text(), "hi"); + assert_eq!( + output.byte_tokens(), + [7, LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL, 11, LLAMA_TOKEN_NULL] + ); + } + + #[test] + fn plain_text_marks_no_byte_with_a_token() { + let output = ChatOutput::from("ab"); + + assert_eq!(output.byte_tokens(), [LLAMA_TOKEN_NULL, LLAMA_TOKEN_NULL]); + } +} diff --git a/llama-cpp-bindings/src/chat_output_completeness.rs b/llama-cpp-bindings/src/chat_output_completeness.rs new file mode 100644 index 000000000..38d7d3024 --- /dev/null +++ b/llama-cpp-bindings/src/chat_output_completeness.rs @@ -0,0 +1,5 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChatOutputCompleteness { + Partial, + Finished, +} diff --git a/llama-cpp-bindings/src/chat_piece_role.rs b/llama-cpp-bindings/src/chat_piece_role.rs new file mode 100644 index 000000000..6c69c497e --- /dev/null +++ b/llama-cpp-bindings/src/chat_piece_role.rs @@ -0,0 +1,16 @@ +use crate::sampled_token_section::SampledTokenSection; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChatPieceRole { + Text(SampledTokenSection), + Marker(SampledTokenSection), +} + +impl ChatPieceRole { + #[must_use] + pub const fn section(self) -> SampledTokenSection { + match self { + Self::Text(section) | Self::Marker(section) => section, + } + } +} diff --git a/llama-cpp-bindings/src/chat_prompt_part.rs b/llama-cpp-bindings/src/chat_prompt_part.rs new file mode 100644 index 000000000..1b8d5f7d7 --- /dev/null +++ b/llama-cpp-bindings/src/chat_prompt_part.rs @@ -0,0 +1,14 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ChatPromptPart { + Template(String), + Input(String), +} + +impl ChatPromptPart { + #[must_use] + pub fn text(&self) -> &str { + match self { + Self::Template(text) | Self::Input(text) => text, + } + } +} diff --git a/llama-cpp-bindings/src/chat_session.rs b/llama-cpp-bindings/src/chat_session.rs new file mode 100644 index 000000000..ed9a359c6 --- /dev/null +++ b/llama-cpp-bindings/src/chat_session.rs @@ -0,0 +1,1152 @@ +use std::ffi::CString; +use std::ffi::c_char; +use std::ptr; +use std::ptr::NonNull; +use std::slice; + +use serde_json::Value; +use serde_json::json; + +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + +use crate::chat_assistant_prefill::ChatAssistantPrefill; +use crate::chat_assistant_turn::ChatAssistantTurn; +use crate::chat_classification::ChatClassification; +use crate::chat_grammar_trigger::ChatGrammarTrigger; +use crate::chat_message_parse_outcome::ChatMessageParseOutcome; +use crate::chat_output::ChatOutput; +use crate::chat_output_completeness::ChatOutputCompleteness; +use crate::chat_prompt_part::ChatPromptPart; +use crate::chat_session_options::ChatSessionOptions; +use crate::chat_span::ChatSpan; +use crate::chat_span_kind::ChatSpanKind; +use crate::chat_spans::ChatSpans; +use crate::chat_templates::ChatTemplates; +use crate::cpp_text_view::cpp_text_view; +use crate::error::chat_session_error::ChatSessionError; +use crate::error::parse_chat_message_error::ParseChatMessageError; +use crate::lazy_grammar_triggers::LazyGrammarTriggers; +use crate::model::LlamaModel; +use crate::model::add_bos::AddBos; +use crate::model::llama_chat_message::LlamaChatMessage; +use crate::model::parse_special_tokens::ParseSpecialTokens; +use crate::parsed_chat::take_parsed_chat_message; +use crate::raw_chat_message::RawChatMessage; +use crate::sampling::LlamaSampler; +use crate::token::LlamaToken; + +const GRAMMAR_ROOT: &str = "root"; + +fn chat_message_json(message: &LlamaChatMessage) -> Result { + Ok(json!({ + "role": message.role.to_str().map_err(ChatSessionError::MessageNotUtf8)?, + "content": message.content.to_str().map_err(ChatSessionError::MessageNotUtf8)?, + })) +} + +fn continued_assistant_message_json(assistant_turn: &ChatAssistantTurn) -> Option { + match assistant_turn { + ChatAssistantTurn::Omitted | ChatAssistantTurn::New => None, + ChatAssistantTurn::ContinueReasoning(reasoning_content) => Some(json!({ + "role": "assistant", + "content": "", + "reasoning_content": reasoning_content, + })), + ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { + reasoning_content, + content, + }) => Some(json!({ + "role": "assistant", + "content": content, + "reasoning_content": reasoning_content, + })), + } +} + +fn messages_json( + messages: &[LlamaChatMessage], + assistant_turn: &ChatAssistantTurn, +) -> Result { + let mut messages = messages + .iter() + .map(chat_message_json) + .collect::, ChatSessionError>>()?; + + messages.extend(continued_assistant_message_json(assistant_turn)); + + CString::new(Value::Array(messages).to_string()).map_err(ChatSessionError::RequestContainsNul) +} + +fn session_text(text: *const c_char, length: usize) -> Result { + unsafe { cpp_text_view(text, length) } + .map(str::to_owned) + .map_err(ChatSessionError::SessionTextNotUtf8) +} + +const fn contract_violation(operation: &'static str, detail: &'static str) -> FfiContractError { + FfiContractError { operation, detail } +} + +fn chat_session_create_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_session_create_status, + out_session: llama_cpp_bindings_sys::llama_rs_chat_session_handle, + out_error: *mut c_char, +) -> Result, ChatSessionError> { + const OPERATION: &str = "llama_rs_chat_session_create"; + let reported_message = || { + unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a failure without an error message", + ) + } + .map_err(ChatSessionError::from) + }; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_OK => NonNull::new(out_session) + .ok_or_else(|| { + contract_violation(OPERATION, "success status contained a null session").into() + }), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON => { + Err(ChatSessionError::ToolsNotJson { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID => { + Err(ChatSessionError::ToolsInvalid { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST => { + Err(ChatSessionError::TemplateCannotServeRequest { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(ChatSessionError::Reported { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED => { + Err(ChatSessionError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ChatSessionError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_TEMPLATES_ARG => { + Err(contract_violation(OPERATION, "was given a null templates argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_VOCAB_ARG => { + Err(contract_violation(OPERATION, "was given a null vocab argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_MESSAGES_JSON_ARG => { + Err(contract_violation(OPERATION, "was given a null messages_json argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG => { + Err(contract_violation(OPERATION, "was given a null tools_json argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG => { + Err(contract_violation(OPERATION, "was given a null out_session argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG => { + Err(contract_violation(OPERATION, "was given a null out_error argument").into()) + } + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn chat_session_parse_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_session_parse_status, + out_handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, + out_error: *mut c_char, + output: &ChatOutput, + completeness: ChatOutputCompleteness, +) -> Result { + const OPERATION: &str = "llama_rs_chat_session_parse"; + let reported_message = || unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a failure without an error message", + ) + }; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OK => NonNull::new(out_handle) + .ok_or_else(|| { + contract_violation(OPERATION, "success status contained a null handle").into() + }) + .and_then(take_parsed_chat_message) + .map(ChatMessageParseOutcome::Recognized), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT => { + Ok(ChatMessageParseOutcome::Unrecognized(RawChatMessage { + text: output.text().to_owned(), + completeness, + ffi_error_message: reported_message()?, + })) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(ParseChatMessageError::Reported { + message: reported_message()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED => { + Err(ParseChatMessageError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ParseChatMessageError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG => { + Err(contract_violation(OPERATION, "was given a null session argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG => { + Err(contract_violation(OPERATION, "was given a null text argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG => { + Err(contract_violation(OPERATION, "was given a null byte_tokens argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG => { + Err(contract_violation(OPERATION, "was given a null out_handle argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG => { + Err(contract_violation(OPERATION, "was given a null out_error argument").into()) + } + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn read_chat_spans( + spans: NonNull, + completeness: ChatOutputCompleteness, +) -> Result { + let view = unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_view_of(spans.as_ptr()) }; + let wrapper_spans = NonNull::new(view.spans.cast_mut()).map_or(&[][..], |first_span| unsafe { + slice::from_raw_parts(first_span.as_ptr(), view.spans_count) + }); + let read = wrapper_spans + .iter() + .map(|wrapper_span| { + ChatSpanKind::try_from(wrapper_span.kind).map(|kind| ChatSpan { + kind, + bytes: wrapper_span.start..wrapper_span.end, + }) + }) + .collect::, FfiStatusError>>() + .map(|spans_read| ChatSpans::new(view.parsed_length, &spans_read, completeness)) + .map_err(ParseChatMessageError::from); + + unsafe { llama_cpp_bindings_sys::llama_rs_chat_spans_free(spans.as_ptr()) }; + + read +} + +fn chat_session_classify_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_session_classify_status, + out_spans: *mut llama_cpp_bindings_sys::llama_rs_chat_spans, + out_error: *mut c_char, + completeness: ChatOutputCompleteness, +) -> Result { + const OPERATION: &str = "llama_rs_chat_session_classify"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OK => NonNull::new(out_spans) + .ok_or_else(|| { + contract_violation(OPERATION, "success status contained null spans").into() + }) + .and_then(|spans| read_chat_spans(spans, completeness)) + .map(ChatClassification::Spans), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT => { + Ok(ChatClassification::OutputDoesNotMatchFormat(completeness)) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(ParseChatMessageError::Reported { + message: unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED => { + Err(ParseChatMessageError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ParseChatMessageError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG => { + Err(contract_violation(OPERATION, "was given a null session argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG => { + Err(contract_violation(OPERATION, "was given a null text argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG => { + Err(contract_violation(OPERATION, "was given a null byte_tokens argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG => { + Err(contract_violation(OPERATION, "was given a null out_spans argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG => { + Err(contract_violation(OPERATION, "was given a null out_error argument").into()) + } + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +/// One chat request rendered by the model's chat template the way llama-server renders it: the +/// prompt, the grammar that constrains tool calls, and the parser of the model's answer. +pub struct ChatSession<'model> { + model: &'model LlamaModel, + session: NonNull, +} + +unsafe impl Send for ChatSession<'_> {} + +impl<'model> ChatSession<'model> { + /// Renders `messages` with `tools_json`, an `OpenAI` tools array, through `templates`: the + /// model's own [`LlamaModel::chat_templates`], or [`ChatTemplates::with_override`]. + /// + /// # Errors + /// + /// Returns the [`ChatSessionError`] describing why the chat template cannot serve the request. + pub fn new( + model: &'model LlamaModel, + templates: &ChatTemplates, + messages: &[LlamaChatMessage], + tools_json: &str, + options: &ChatSessionOptions, + ) -> Result { + let messages_json = messages_json(messages, &options.assistant_turn)?; + let tools_json = CString::new(tools_json).map_err(ChatSessionError::RequestContainsNul)?; + let mut out_session: llama_cpp_bindings_sys::llama_rs_chat_session_handle = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_create( + templates.as_ptr(), + model.vocab_ptr(), + messages_json.as_ptr(), + tools_json.as_ptr(), + options.into(), + &raw mut out_session, + &raw mut out_error, + ) + }; + + chat_session_create_status_to_result(status, out_session, out_error) + .map(|session| Self { model, session }) + } + + fn text_of( + &self, + read: impl FnOnce( + *mut llama_cpp_bindings_sys::llama_rs_chat_session, + *mut usize, + ) -> *const c_char, + ) -> Result { + let mut length = 0; + let text = read(self.session.as_ptr(), &raw mut length); + + session_text(text, length) + } + + fn texts_of( + &self, + count: usize, + read: impl Fn( + *mut llama_cpp_bindings_sys::llama_rs_chat_session, + usize, + *mut usize, + ) -> *const c_char, + ) -> Result, ChatSessionError> { + (0..count) + .map(|index| self.text_of(|session, out_length| read(session, index, out_length))) + .collect() + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the rendered prompt is not UTF-8. + pub fn prompt_parts(&self) -> Result, ChatSessionError> { + let part_count = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_prompt_part_count(self.session.as_ptr()) + }; + + (0..part_count) + .map(|index| { + let text = self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_prompt_part_text( + session, index, out_length, + ) + })?; + let is_input = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_prompt_part_is_input( + self.session.as_ptr(), + index, + ) + }; + + Ok(if is_input { + ChatPromptPart::Input(text) + } else { + ChatPromptPart::Template(text) + }) + }) + .collect() + } + + /// The rendered prompt as one string, for [`crate::mtmd::mtmd_context::MtmdContext::tokenize`], which + /// finds the media markers in it the way llama-server hands a multimodal prompt to mtmd. + /// + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the rendered prompt is not UTF-8. + pub fn prompt(&self) -> Result { + Ok(self + .prompt_parts()? + .iter() + .map(ChatPromptPart::text) + .collect()) + } + + /// Tokenizes the rendered prompt part by part: control tokens are recognized in the + /// template's own text but never in user input, and BOS is added once before the first part + /// when the vocabulary asks for it. + /// + /// # Errors + /// + /// Returns the [`ChatSessionError`] of reading or tokenizing a prompt part. + pub fn tokenize_prompt(&self) -> Result, ChatSessionError> { + let mut tokens = Vec::new(); + + for (part_index, part) in self.prompt_parts()?.iter().enumerate() { + let add_bos = if part_index == 0 { + AddBos::Always + } else { + AddBos::Never + }; + let (text, parse_special) = match part { + ChatPromptPart::Template(text) => (text, ParseSpecialTokens::Always), + ChatPromptPart::Input(text) => (text, ParseSpecialTokens::Never), + }; + + tokens.extend(self.model.str_to_token(text, add_bos, parse_special)?); + } + + Ok(tokens) + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the generation prompt is not UTF-8. + pub fn generation_prompt(&self) -> Result { + self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_generation_prompt(session, out_length) + }) + } + + #[must_use] + pub fn generation_prompt_tokens(&self) -> Vec { + let count = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_generation_prompt_token_count( + self.session.as_ptr(), + ) + }; + + (0..count) + .map(|index| { + LlamaToken::new(unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_generation_prompt_token( + self.session.as_ptr(), + index, + ) + }) + }) + .collect() + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the grammar is not UTF-8. + pub fn grammar(&self) -> Result { + self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar(session, out_length) + }) + } + + #[must_use] + pub fn grammar_is_lazy(&self) -> bool { + unsafe { llama_cpp_bindings_sys::llama_rs_chat_session_grammar_lazy(self.session.as_ptr()) } + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when a trigger is not UTF-8, or + /// [`ChatSessionError::FfiStatus`] for a trigger type unknown to this binding. + pub fn grammar_triggers(&self) -> Result, ChatSessionError> { + let count = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar_trigger_count( + self.session.as_ptr(), + ) + }; + + (0..count) + .map(|index| { + let trigger_type = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar_trigger_type( + self.session.as_ptr(), + index, + ) + }; + let value = self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar_trigger_value( + session, index, out_length, + ) + })?; + + Ok(ChatGrammarTrigger::from_wrapper( + trigger_type, + value, + || { + LlamaToken::new(unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_grammar_trigger_token( + self.session.as_ptr(), + index, + ) + }) + }, + )?) + }) + .collect() + } + + #[must_use] + pub fn preserved_tokens(&self) -> Vec { + let count = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_preserved_token_count( + self.session.as_ptr(), + ) + }; + + (0..count) + .map(|index| { + LlamaToken::new(unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_preserved_token( + self.session.as_ptr(), + index, + ) + }) + }) + .collect() + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when the tag is not UTF-8. + pub fn thinking_start_tag(&self) -> Result { + self.text_of(|session, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_thinking_start_tag(session, out_length) + }) + } + + /// # Errors + /// + /// Returns [`ChatSessionError::SessionTextNotUtf8`] when a tag is not UTF-8. + pub fn thinking_end_tags(&self) -> Result, ChatSessionError> { + self.texts_of( + unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_thinking_end_tag_count( + self.session.as_ptr(), + ) + }, + |session, index, out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_thinking_end_tag( + session, index, out_length, + ) + }, + ) + } + + #[must_use] + pub const fn model(&self) -> &'model LlamaModel { + self.model + } + + /// The sampler that constrains generation to the request's tool calls, or `None` when the + /// request needs no grammar. A grammar that is not lazy starts after the generation prompt, + /// so it accepts the generation-prompt tokens first. + /// + /// # Errors + /// + /// Returns the [`ChatSessionError`] of reading the grammar, escaping a trigger word, building + /// the sampler, or accepting a generation-prompt token. + pub fn grammar_sampler(&self) -> Result, ChatSessionError> { + let grammar = self.grammar()?; + + if grammar.is_empty() { + return Ok(None); + } + + if !self.grammar_is_lazy() { + let mut sampler = LlamaSampler::grammar(self.model, &grammar, GRAMMAR_ROOT)?; + + for token in self.generation_prompt_tokens() { + sampler.accept(token)?; + } + + return Ok(Some(sampler)); + } + + let LazyGrammarTriggers { patterns, tokens } = + LazyGrammarTriggers::of(self.grammar_triggers()?)?; + + Ok(Some(LlamaSampler::grammar_lazy( + self.model, + &grammar, + GRAMMAR_ROOT, + &patterns, + &tokens, + )?)) + } + + /// Parses the model's answer with the template's parser. + /// + /// # Errors + /// + /// Returns [`ParseChatMessageError`] when llama.cpp fails while parsing; an answer that does + /// not match the template's format is [`ChatMessageParseOutcome::Unrecognized`]. + pub fn parse( + &self, + output: &ChatOutput, + completeness: ChatOutputCompleteness, + ) -> Result { + let mut out_handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_parse( + self.session.as_ptr(), + output.text().as_ptr().cast::(), + output.text().len(), + output.byte_tokens().as_ptr(), + completeness == ChatOutputCompleteness::Partial, + &raw mut out_handle, + &raw mut out_error, + ) + }; + + chat_session_parse_status_to_result(status, out_handle, out_error, output, completeness) + } + + /// Returns how the template's parser attributes the bytes of the model's answer to + /// reasoning, content, and tool calls. + /// + /// # Errors + /// + /// Returns [`ParseChatMessageError`] when llama.cpp fails while parsing. + pub fn classify( + &self, + output: &ChatOutput, + completeness: ChatOutputCompleteness, + ) -> Result { + let mut out_spans: *mut llama_cpp_bindings_sys::llama_rs_chat_spans = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_session_classify( + self.session.as_ptr(), + output.text().as_ptr().cast::(), + output.text().len(), + output.byte_tokens().as_ptr(), + completeness == ChatOutputCompleteness::Partial, + &raw mut out_spans, + &raw mut out_error, + ) + }; + + chat_session_classify_status_to_result(status, out_spans, out_error, completeness) + } +} + +impl Drop for ChatSession<'_> { + fn drop(&mut self) { + unsafe { llama_cpp_bindings_sys::llama_rs_chat_session_free(self.session.as_ptr()) }; + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::ffi::CStr; + use std::ffi::CString; + use std::ptr; + + use serde_json::json; + + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::chat_session_classify_status_to_result; + use super::chat_session_create_status_to_result; + use super::chat_session_parse_status_to_result; + use super::messages_json; + use crate::chat_assistant_prefill::ChatAssistantPrefill; + use crate::chat_assistant_turn::ChatAssistantTurn; + use crate::chat_classification::ChatClassification; + use crate::chat_message_parse_outcome::ChatMessageParseOutcome; + use crate::chat_output::ChatOutput; + use crate::chat_output_completeness::ChatOutputCompleteness; + use crate::error::chat_session_error::ChatSessionError; + use crate::error::parse_chat_message_error::ParseChatMessageError; + use crate::model::llama_chat_message::LlamaChatMessage; + use crate::raw_chat_message::RawChatMessage; + + type CreateRefusal = ( + llama_cpp_bindings_sys::llama_rs_chat_session_create_status, + fn(String) -> ChatSessionError, + ); + + fn parsed_messages(messages_json: &CStr) -> serde_json::Value { + serde_json::from_str(messages_json.to_str().unwrap()).unwrap() + } + + #[test] + fn creating_a_session_reports_each_null_argument_and_a_null_created_session_as_a_contract_violation() + { + for (status, detail) in [ + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_TEMPLATES_ARG, + "was given a null templates argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_VOCAB_ARG, + "was given a null vocab argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_MESSAGES_JSON_ARG, + "was given a null messages_json argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_TOOLS_JSON_ARG, + "was given a null tools_json argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_SESSION_ARG, + "was given a null out_session argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_NULL_OUT_ERROR_ARG, + "was given a null out_error argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_OK, + "success status contained a null session", + ), + ] { + assert_eq!( + chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()) + .err(), + Some(ChatSessionError::FfiContract(FfiContractError { + operation: "llama_rs_chat_session_create", + detail + })) + ); + } + } + + #[test] + fn creating_a_session_reports_why_llama_cpp_refused_the_request() { + let refusals: [CreateRefusal; 4] = [ + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, + |message| ChatSessionError::ToolsNotJson { message }, + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, + |message| ChatSessionError::ToolsInvalid { message }, + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, + |message| ChatSessionError::TemplateCannotServeRequest { message }, + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, + |message| ChatSessionError::Reported { message }, + ), + ]; + + for (status, expected_error) in refusals { + assert_eq!( + chat_session_create_status_to_result(status, ptr::null_mut(), unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"the reason".as_ptr()) + }) + .err(), + Some(expected_error("the reason".to_owned())) + ); + } + } + + #[test] + fn creating_a_session_reports_a_refusal_without_a_message_as_a_contract_violation() { + let refusals = [ + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_NOT_JSON, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TOOLS_INVALID, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_TEMPLATE_CANNOT_SERVE_REQUEST, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ]; + + assert_eq!( + refusals.map(|status| { + chat_session_create_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() + }), + refusals.map(|_status| { + Some(ChatSessionError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_chat_session_create", + detail: "reported a failure without an error message", + }, + ))) + }) + ); + } + + #[test] + fn creating_a_session_reports_memory_exhaustion() { + assert_eq!( + [ + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CREATE_LLAMA_CPP_OUT_OF_MEMORY, + ] + .map(|status| chat_session_create_status_to_result( + status, + ptr::null_mut(), + ptr::null_mut() + ) + .err()), + [ + Some(ChatSessionError::NotEnoughMemory), + Some(ChatSessionError::LlamaCppOutOfMemory), + ] + ); + } + + #[test] + fn creating_a_session_preserves_an_unknown_status() { + assert_eq!( + chat_session_create_status_to_result(255, ptr::null_mut(), ptr::null_mut()).err(), + Some(ChatSessionError::FfiStatus(FfiStatusError { + operation: "llama_rs_chat_session_create", + code: 255 + })) + ); + } + + #[test] + fn parsing_an_answer_that_does_not_match_the_format_keeps_the_raw_answer() { + let output = ChatOutput::from("not the format"); + + assert_eq!( + chat_session_parse_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, + ptr::null_mut(), + unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"the format differs".as_ptr()) + }, + &output, + ChatOutputCompleteness::Finished, + ), + Ok(ChatMessageParseOutcome::Unrecognized(RawChatMessage { + text: "not the format".to_owned(), + completeness: ChatOutputCompleteness::Finished, + ffi_error_message: "the format differs".to_owned(), + })) + ); + } + + #[test] + fn parsing_an_answer_maps_every_failure_status() { + let output = ChatOutput::default(); + let parse = |status, out_error| { + chat_session_parse_status_to_result( + status, + ptr::null_mut(), + out_error, + &output, + ChatOutputCompleteness::Partial, + ) + .err() + }; + let contract = |detail| { + Some(ParseChatMessageError::FfiContract(FfiContractError { + operation: "llama_rs_chat_session_parse", + detail, + })) + }; + + assert_eq!( + [ + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OK, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"the parser threw".as_ptr()) } + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_SESSION_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_TEXT_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_BYTE_TOKENS_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_HANDLE_ARG, + ptr::null_mut() + ), + parse( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_NULL_OUT_ERROR_ARG, + ptr::null_mut() + ), + parse(255, ptr::null_mut()), + ], + [ + contract("success status contained a null handle"), + Some(ParseChatMessageError::Reported { + message: "the parser threw".to_owned() + }), + Some(ParseChatMessageError::NotEnoughMemory), + Some(ParseChatMessageError::LlamaCppOutOfMemory), + contract("was given a null session argument"), + contract("was given a null text argument"), + contract("was given a null byte_tokens argument"), + contract("was given a null out_handle argument"), + contract("was given a null out_error argument"), + Some(ParseChatMessageError::FfiStatus(FfiStatusError { operation: "llama_rs_chat_session_parse", code: 255 })), + ] + ); + } + + #[test] + fn classifying_an_answer_that_does_not_match_the_format_is_not_an_error() { + assert_eq!( + chat_session_classify_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OUTPUT_DOES_NOT_MATCH_FORMAT, + ptr::null_mut(), + ptr::null_mut(), + ChatOutputCompleteness::Partial, + ), + Ok(ChatClassification::OutputDoesNotMatchFormat( + ChatOutputCompleteness::Partial + )) + ); + } + + #[test] + fn classifying_an_answer_maps_every_failure_status() { + let classify = |status, out_error| { + chat_session_classify_status_to_result( + status, + ptr::null_mut(), + out_error, + ChatOutputCompleteness::Partial, + ) + .err() + }; + let contract = |detail| { + Some(ParseChatMessageError::FfiContract(FfiContractError { + operation: "llama_rs_chat_session_classify", + detail, + })) + }; + + assert_eq!( + [ + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_OK, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"the parser threw".as_ptr()) } + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_SESSION_ARG, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_TEXT_ARG, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_BYTE_TOKENS_ARG, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_SPANS_ARG, + ptr::null_mut() + ), + classify( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_NULL_OUT_ERROR_ARG, + ptr::null_mut() + ), + classify(255, ptr::null_mut()), + ], + [ + contract("success status contained null spans"), + Some(ParseChatMessageError::Reported { + message: "the parser threw".to_owned() + }), + Some(ParseChatMessageError::NotEnoughMemory), + Some(ParseChatMessageError::LlamaCppOutOfMemory), + contract("was given a null session argument"), + contract("was given a null text argument"), + contract("was given a null byte_tokens argument"), + contract("was given a null out_spans argument"), + contract("was given a null out_error argument"), + Some(ParseChatMessageError::FfiStatus(FfiStatusError { operation: "llama_rs_chat_session_classify", code: 255 })), + ] + ); + } + + #[test] + fn a_new_assistant_turn_adds_no_message() { + let messages = [LlamaChatMessage::new("user".to_owned(), "hi".to_owned()).unwrap()]; + + assert_eq!( + parsed_messages(&messages_json(&messages, &ChatAssistantTurn::New).unwrap()), + json!([{"role": "user", "content": "hi"}]) + ); + } + + #[test] + fn a_message_whose_role_or_content_is_not_utf8_is_refused() { + let not_utf8 = CString::new([0xff]).unwrap(); + let utf8_error = not_utf8.to_str().err(); + let messages = [ + LlamaChatMessage { + role: not_utf8.clone(), + content: c"hi".to_owned(), + }, + LlamaChatMessage { + role: c"user".to_owned(), + content: not_utf8, + }, + ]; + + assert_eq!( + messages.map(|message| messages_json(&[message], &ChatAssistantTurn::New).err()), + [utf8_error, utf8_error].map(|error| error.map(ChatSessionError::MessageNotUtf8)) + ); + } + + #[test] + fn continuing_reasoning_appends_an_assistant_message_with_that_reasoning() { + assert_eq!( + parsed_messages( + &messages_json(&[], &ChatAssistantTurn::ContinueReasoning("hmm".to_owned())) + .unwrap() + ), + json!([{"role": "assistant", "content": "", "reasoning_content": "hmm"}]) + ); + } + + #[test] + fn continuing_content_appends_an_assistant_message_with_the_prefill() { + assert_eq!( + parsed_messages( + &messages_json( + &[], + &ChatAssistantTurn::ContinueContent(ChatAssistantPrefill { + reasoning_content: "done".to_owned(), + content: "The".to_owned(), + }) + ) + .unwrap() + ), + json!([{"role": "assistant", "content": "The", "reasoning_content": "done"}]) + ); + } + + #[test] + fn a_parse_or_classification_failure_without_a_message_is_a_contract_violation() { + let output = ChatOutput::default(); + + assert_eq!( + [ + chat_session_parse_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_OUTPUT_DOES_NOT_MATCH_FORMAT, + ptr::null_mut(), + ptr::null_mut(), + &output, + ChatOutputCompleteness::Finished, + ) + .err(), + chat_session_parse_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_PARSE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ptr::null_mut(), + &output, + ChatOutputCompleteness::Finished, + ) + .err(), + chat_session_classify_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SESSION_CLASSIFY_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ptr::null_mut(), + ChatOutputCompleteness::Finished, + ) + .err(), + ], + [ + Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { + operation: "llama_rs_chat_session_parse", + detail: "reported a failure without an error message", + }))), + Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { + operation: "llama_rs_chat_session_parse", + detail: "reported a failure without an error message", + }))), + Some(ParseChatMessageError::CppString(CppStringError::Missing(FfiContractError { + operation: "llama_rs_chat_session_classify", + detail: "reported a thrown C++ exception without an error message", + }))), + ] + ); + } +} diff --git a/llama-cpp-bindings/src/chat_session_attribution.rs b/llama-cpp-bindings/src/chat_session_attribution.rs new file mode 100644 index 000000000..da6b294ef --- /dev/null +++ b/llama-cpp-bindings/src/chat_session_attribution.rs @@ -0,0 +1,13 @@ +use std::collections::VecDeque; + +use crate::chat_output::ChatOutput; +use crate::chat_session::ChatSession; +use crate::pending_chat_piece::PendingChatPiece; +use crate::token::LlamaToken; + +pub struct ChatSessionAttribution<'session, 'model> { + pub session: &'session ChatSession<'model>, + pub preserved_tokens: Vec, + pub output: ChatOutput, + pub pending_pieces: VecDeque, +} diff --git a/llama-cpp-bindings/src/chat_session_options.rs b/llama-cpp-bindings/src/chat_session_options.rs new file mode 100644 index 000000000..d4a4835f1 --- /dev/null +++ b/llama-cpp-bindings/src/chat_session_options.rs @@ -0,0 +1,28 @@ +use crate::chat_assistant_turn::ChatAssistantTurn; +use crate::chat_tool_choice::ChatToolChoice; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatSessionOptions { + pub assistant_turn: ChatAssistantTurn, + pub enable_thinking: bool, + pub parallel_tool_calls: bool, + pub tool_choice: ChatToolChoice, +} + +impl From<&ChatSessionOptions> for llama_cpp_bindings_sys::llama_rs_chat_session_options { + fn from( + ChatSessionOptions { + assistant_turn, + enable_thinking, + parallel_tool_calls, + tool_choice, + }: &ChatSessionOptions, + ) -> Self { + Self { + assistant_turn: assistant_turn.into(), + enable_thinking: *enable_thinking, + parallel_tool_calls: *parallel_tool_calls, + tool_choice: (*tool_choice).into(), + } + } +} diff --git a/llama-cpp-bindings/src/chat_span.rs b/llama-cpp-bindings/src/chat_span.rs new file mode 100644 index 000000000..e61a214f8 --- /dev/null +++ b/llama-cpp-bindings/src/chat_span.rs @@ -0,0 +1,9 @@ +use std::ops::Range; + +use crate::chat_span_kind::ChatSpanKind; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatSpan { + pub kind: ChatSpanKind, + pub bytes: Range, +} diff --git a/llama-cpp-bindings/src/chat_span_kind.rs b/llama-cpp-bindings/src/chat_span_kind.rs new file mode 100644 index 000000000..1048a5306 --- /dev/null +++ b/llama-cpp-bindings/src/chat_span_kind.rs @@ -0,0 +1,120 @@ +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::chat_piece_role::ChatPieceRole; +use crate::sampled_token_section::SampledTokenSection; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChatSpanKind { + ReasoningBlock, + Reasoning, + Content, + ToolCall, + ToolCallMarker, +} + +impl ChatSpanKind { + #[must_use] + pub const fn piece_role(self) -> ChatPieceRole { + match self { + Self::ReasoningBlock => ChatPieceRole::Marker(SampledTokenSection::Reasoning), + Self::Reasoning => ChatPieceRole::Text(SampledTokenSection::Reasoning), + Self::Content => ChatPieceRole::Text(SampledTokenSection::Content), + Self::ToolCall => ChatPieceRole::Text(SampledTokenSection::ToolCall), + Self::ToolCallMarker => ChatPieceRole::Marker(SampledTokenSection::ToolCall), + } + } +} + +impl TryFrom for ChatSpanKind { + type Error = FfiStatusError; + + fn try_from( + kind: llama_cpp_bindings_sys::llama_rs_chat_span_kind, + ) -> Result { + match kind { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_REASONING_BLOCK => { + Ok(Self::ReasoningBlock) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_REASONING => Ok(Self::Reasoning), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_CONTENT => Ok(Self::Content), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL => Ok(Self::ToolCall), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER => { + Ok(Self::ToolCallMarker) + } + other => Err(FfiStatusError { + operation: "llama_rs_chat_spans_view_of", + code: i64::from(other), + }), + } + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::ChatSpanKind; + use crate::chat_piece_role::ChatPieceRole; + use crate::sampled_token_section::SampledTokenSection; + + #[test] + fn text_spans_attribute_their_bytes_as_visible_text_of_their_section() { + assert_eq!( + [ + ChatSpanKind::Reasoning, + ChatSpanKind::Content, + ChatSpanKind::ToolCall + ] + .map(ChatSpanKind::piece_role), + [ + ChatPieceRole::Text(SampledTokenSection::Reasoning), + ChatPieceRole::Text(SampledTokenSection::Content), + ChatPieceRole::Text(SampledTokenSection::ToolCall), + ] + ); + } + + #[test] + fn marker_spans_attribute_their_bytes_as_markers_of_their_section() { + assert_eq!( + [ChatSpanKind::ReasoningBlock, ChatSpanKind::ToolCallMarker] + .map(ChatSpanKind::piece_role), + [ + ChatPieceRole::Marker(SampledTokenSection::Reasoning), + ChatPieceRole::Marker(SampledTokenSection::ToolCall), + ] + ); + } + + #[test] + fn every_wrapper_span_kind_maps_to_its_variant() { + assert_eq!( + [ + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_REASONING_BLOCK, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_REASONING, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_CONTENT, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL, + llama_cpp_bindings_sys::LLAMA_RS_CHAT_SPAN_KIND_TOOL_CALL_MARKER, + ] + .map(ChatSpanKind::try_from), + [ + Ok(ChatSpanKind::ReasoningBlock), + Ok(ChatSpanKind::Reasoning), + Ok(ChatSpanKind::Content), + Ok(ChatSpanKind::ToolCall), + Ok(ChatSpanKind::ToolCallMarker), + ] + ); + } + + #[test] + fn an_unknown_wrapper_span_kind_is_a_status_error() { + assert_eq!( + ChatSpanKind::try_from(99), + Err(FfiStatusError { + operation: "llama_rs_chat_spans_view_of", + code: 99, + }) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_span_segment.rs b/llama-cpp-bindings/src/chat_span_segment.rs new file mode 100644 index 000000000..3732a42eb --- /dev/null +++ b/llama-cpp-bindings/src/chat_span_segment.rs @@ -0,0 +1,9 @@ +use std::ops::Range; + +use crate::chat_piece_role::ChatPieceRole; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatSpanSegment { + pub bytes: Range, + pub role: ChatPieceRole, +} diff --git a/llama-cpp-bindings/src/chat_spans.rs b/llama-cpp-bindings/src/chat_spans.rs new file mode 100644 index 000000000..5270e0198 --- /dev/null +++ b/llama-cpp-bindings/src/chat_spans.rs @@ -0,0 +1,302 @@ +use std::cmp::Reverse; +use std::ops::Range; + +use crate::chat_output_completeness::ChatOutputCompleteness; +use crate::chat_piece_role::ChatPieceRole; +use crate::chat_span::ChatSpan; +use crate::chat_span_segment::ChatSpanSegment; +use crate::sampled_token_section::SampledTokenSection; + +fn innermost_role_at(spans: &[ChatSpan], byte: usize) -> Option { + spans + .iter() + .enumerate() + .filter(|(_order, span)| span.bytes.contains(&byte)) + .min_by_key(|(order, span)| (span.bytes.len(), Reverse(*order))) + .map(|(_order, span)| span.kind.piece_role()) +} + +fn segments_of(spans: &[ChatSpan]) -> Vec { + let mut boundaries = spans + .iter() + .flat_map(|span| [span.bytes.start, span.bytes.end]) + .collect::>(); + + boundaries.sort_unstable(); + boundaries.dedup(); + + boundaries + .iter() + .zip(boundaries.iter().skip(1)) + .filter_map(|(&start, &end)| { + innermost_role_at(spans, start).map(|role| ChatSpanSegment { + bytes: start..end, + role, + }) + }) + .collect() +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChatSpans { + parsed_length: usize, + segments: Vec, + completeness: ChatOutputCompleteness, +} + +impl ChatSpans { + /// Splits the nested `spans` of llama.cpp's parse once into ordered segments, each carrying + /// the role of the innermost span over it, so roles are looked up by binary search. + #[must_use] + pub fn new( + parsed_length: usize, + spans: &[ChatSpan], + completeness: ChatOutputCompleteness, + ) -> Self { + Self { + parsed_length, + segments: segments_of(spans), + completeness, + } + } + + fn first_segment_ending_after(&self, byte: usize) -> usize { + self.segments + .partition_point(|segment| segment.bytes.end <= byte) + } + + fn tagged_role_at(&self, byte: usize) -> Option { + self.segments + .get(self.first_segment_ending_after(byte)) + .filter(|segment| segment.bytes.contains(&byte)) + .map(|segment| segment.role) + } + + fn untagged_marker_section(&self, byte: usize) -> Option { + let next = self + .segments + .get(self.first_segment_ending_after(byte)) + .filter(|segment| segment.bytes.start.max(byte) < self.parsed_length) + .map(|segment| segment.role.section()); + + if next.is_none() && self.completeness == ChatOutputCompleteness::Partial { + return None; + } + + let previous = self + .segments + .partition_point(|segment| segment.bytes.start < byte) + .checked_sub(1) + .and_then(|index| self.segments.get(index)) + .map(|segment| segment.role.section()); + + Some(match (previous, next) { + (Some(SampledTokenSection::Reasoning), _) + | (_, Some(SampledTokenSection::Reasoning)) => SampledTokenSection::Reasoning, + (Some(SampledTokenSection::ToolCall), _) | (_, Some(SampledTokenSection::ToolCall)) => { + SampledTokenSection::ToolCall + } + _ => SampledTokenSection::Content, + }) + } + + /// Returns how llama.cpp's parse attributes the token whose text occupies `bytes` of the + /// output, or `None` while the parse has not settled the token yet. + #[must_use] + pub fn classify_token(&self, bytes: &Range) -> Option { + if self.completeness == ChatOutputCompleteness::Partial && bytes.end > self.parsed_length { + return None; + } + + self.segments + .iter() + .skip(self.first_segment_ending_after(bytes.start)) + .take_while(|segment| { + segment.bytes.start.max(bytes.start) < segment.bytes.end.min(bytes.end) + }) + .map(|segment| segment.role) + .find(|role| matches!(role, ChatPieceRole::Text(_))) + .or_else(|| { + bytes + .clone() + .next() + .and_then(|first_byte| self.tagged_role_at(first_byte)) + }) + .or_else(|| { + self.untagged_marker_section(bytes.start) + .map(ChatPieceRole::Marker) + }) + } +} + +#[cfg(test)] +mod tests { + use super::ChatSpans; + use crate::chat_output_completeness::ChatOutputCompleteness; + use crate::chat_piece_role::ChatPieceRole; + use crate::chat_span::ChatSpan; + use crate::chat_span_kind::ChatSpanKind; + use crate::sampled_token_section::SampledTokenSection; + + fn span(kind: ChatSpanKind, start: usize, end: usize) -> ChatSpan { + ChatSpan { + kind, + bytes: start..end, + } + } + + fn reasoning_then_content(completeness: ChatOutputCompleteness) -> ChatSpans { + ChatSpans::new( + 30, + &[ + span(ChatSpanKind::Reasoning, 7, 12), + span(ChatSpanKind::Content, 22, 30), + ], + completeness, + ) + } + + #[test] + fn a_token_inside_reasoning_text_is_visible_reasoning() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(7..12)), + Some(ChatPieceRole::Text(SampledTokenSection::Reasoning)) + ); + } + + #[test] + fn an_untagged_token_before_reasoning_opens_reasoning() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(0..7)), + Some(ChatPieceRole::Marker(SampledTokenSection::Reasoning)) + ); + } + + #[test] + fn an_untagged_token_after_reasoning_closes_reasoning() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(12..22)), + Some(ChatPieceRole::Marker(SampledTokenSection::Reasoning)) + ); + } + + #[test] + fn a_token_partly_in_content_text_is_visible_content() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(20..24)), + Some(ChatPieceRole::Text(SampledTokenSection::Content)) + ); + } + + #[test] + fn a_token_past_the_partial_parse_waits_for_more_output() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(28..31)), + None + ); + } + + #[test] + fn the_innermost_span_decides_a_byte_inside_nested_spans() { + let spans = ChatSpans::new( + 20, + &[ + span(ChatSpanKind::ToolCall, 0, 20), + span(ChatSpanKind::ToolCallMarker, 0, 6), + ], + ChatOutputCompleteness::Partial, + ); + + assert_eq!( + [ + spans.classify_token(&(0..6)), + spans.classify_token(&(6..10)) + ], + [ + Some(ChatPieceRole::Marker(SampledTokenSection::ToolCall)), + Some(ChatPieceRole::Text(SampledTokenSection::ToolCall)), + ] + ); + } + + #[test] + fn an_untagged_token_beside_a_tool_call_marks_the_tool_call() { + let spans = ChatSpans::new( + 20, + &[ + span(ChatSpanKind::Content, 0, 5), + span(ChatSpanKind::ToolCall, 10, 20), + ], + ChatOutputCompleteness::Partial, + ); + + assert_eq!( + spans.classify_token(&(5..10)), + Some(ChatPieceRole::Marker(SampledTokenSection::ToolCall)) + ); + } + + #[test] + fn an_untagged_tail_of_a_partial_parse_waits_for_what_follows() { + let spans = ChatSpans::new( + 8, + &[span(ChatSpanKind::Content, 0, 5)], + ChatOutputCompleteness::Partial, + ); + + assert_eq!(spans.classify_token(&(5..8)), None); + } + + #[test] + fn an_untagged_tail_of_a_finished_parse_after_content_is_a_content_marker() { + let spans = ChatSpans::new( + 8, + &[span(ChatSpanKind::Content, 0, 5)], + ChatOutputCompleteness::Finished, + ); + + assert_eq!( + spans.classify_token(&(5..8)), + Some(ChatPieceRole::Marker(SampledTokenSection::Content)) + ); + } + + #[test] + fn a_token_without_text_right_after_reasoning_is_a_reasoning_marker() { + assert_eq!( + reasoning_then_content(ChatOutputCompleteness::Partial).classify_token(&(12..12)), + Some(ChatPieceRole::Marker(SampledTokenSection::Reasoning)) + ); + } + + #[test] + fn a_token_without_text_inside_a_text_span_is_a_marker_of_that_section() { + let spans = ChatSpans::new( + 10, + &[span(ChatSpanKind::Content, 0, 10)], + ChatOutputCompleteness::Finished, + ); + + assert_eq!( + spans.classify_token(&(4..4)), + Some(ChatPieceRole::Marker(SampledTokenSection::Content)) + ); + } + + #[test] + fn a_reasoning_block_byte_outside_reasoning_text_is_a_reasoning_marker() { + let spans = ChatSpans::new( + 20, + &[ + span(ChatSpanKind::ReasoningBlock, 0, 20), + span(ChatSpanKind::Reasoning, 7, 12), + ], + ChatOutputCompleteness::Partial, + ); + + assert_eq!( + spans.classify_token(&(12..20)), + Some(ChatPieceRole::Marker(SampledTokenSection::Reasoning)) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls.rs b/llama-cpp-bindings/src/chat_template_tool_calls.rs deleted file mode 100644 index a1b5010e9..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls.rs +++ /dev/null @@ -1,11 +0,0 @@ -pub mod detect; -pub mod gemma4_call_block; -pub mod glm47_key_value_tags; -pub mod known_marker_candidates; -pub mod mistral3_arrow_args; -pub mod qwen3_json_inside_tool_call; -pub mod qwen_xml_tags; -pub mod tool_call_format_registration; - -pub use detect::detect; -pub use known_marker_candidates::known_marker_candidates; diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/detect.rs b/llama-cpp-bindings/src/chat_template_tool_calls/detect.rs deleted file mode 100644 index 422e4ea7d..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/detect.rs +++ /dev/null @@ -1,47 +0,0 @@ -use llama_cpp_bindings_types::ToolCallMarkers; - -use crate::chat_template_tool_calls::tool_call_format_registration::ToolCallFormatRegistration; - -#[must_use] -pub fn detect(template: &str) -> Option { - ToolCallFormatRegistration::KNOWN - .iter() - .find_map(|registration| (registration.detect)(template)) -} - -#[cfg(test)] -mod tests { - use super::detect; - use crate::chat_template_tool_calls::gemma4_call_block::Gemma4CallBlockFormat; - use crate::chat_template_tool_calls::mistral3_arrow_args::Mistral3ArrowArgsFormat; - use crate::chat_template_tool_calls::qwen_xml_tags::QwenXmlTagsFormat; - - #[test] - fn detects_gemma4_call_block_format() { - let template = "{{- '<|tool_call>call:' + function['name'] + '{' -}}"; - let markers = detect(template).expect("must dispatch to Gemma 4"); - - assert_eq!(markers, Gemma4CallBlockFormat::markers()); - } - - #[test] - fn detects_mistral3_arrow_args_format() { - let template = "{{- name + '[ARGS]' + arguments }}"; - let markers = detect(template).expect("must dispatch to Mistral 3"); - - assert_eq!(markers, Mistral3ArrowArgsFormat::markers()); - } - - #[test] - fn detects_qwen_xml_tags_format() { - let template = "{{- '\\n\\n' }}"; - let markers = detect(template).expect("must dispatch to Qwen XML tags"); - - assert_eq!(markers, QwenXmlTagsFormat::markers()); - } - - #[test] - fn returns_none_when_no_known_format_matches() { - assert!(detect("plain unrelated template").is_none()); - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs b/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs deleted file mode 100644 index 978ddeea3..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/gemma4_call_block.rs +++ /dev/null @@ -1,79 +0,0 @@ -use llama_cpp_bindings_types::PairedQuoteShape; -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; -use llama_cpp_bindings_types::ToolCallValueQuote; - -pub struct Gemma4CallBlockFormat; - -impl Gemma4CallBlockFormat { - const TEMPLATE_FINGERPRINT: &'static str = "'<|tool_call>call:'"; - - #[must_use] - pub fn markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "<|tool_call>call:".to_owned(), - close: "}".to_owned(), - args_shape: ToolCallArgsShape::PairedQuote(PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }), - } - } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::Gemma4CallBlockFormat; - - #[test] - fn detects_gemma4_template_with_tool_call_call_literal() { - use llama_cpp_bindings_types::PairedQuoteShape; - use llama_cpp_bindings_types::ToolCallValueQuote; - - let template = "...{{- '<|tool_call>call:' + function['name'] + '{' -}}..."; - let markers = - Gemma4CallBlockFormat::detect(template).expect("Gemma 4 template must be detected"); - - assert_eq!(markers.open, "<|tool_call>call:"); - assert_eq!(markers.close, "}"); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::PairedQuote(PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(Gemma4CallBlockFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(Gemma4CallBlockFormat::detect("").is_none()); - } - - #[test] - fn returns_none_when_fingerprint_substring_appears_without_jinja_apostrophes() { - let template = "doc explaining the <|tool_call>call: format in prose, not as a literal"; - assert!(Gemma4CallBlockFormat::detect(template).is_none()); - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs b/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs deleted file mode 100644 index 5e1627f2a..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/glm47_key_value_tags.rs +++ /dev/null @@ -1,68 +0,0 @@ -use llama_cpp_bindings_types::KeyValueXmlTagsShape; -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; - -pub struct Glm47KeyValueTagsFormat; - -impl Glm47KeyValueTagsFormat { - const TEMPLATE_FINGERPRINT: &'static str = ""; - - #[must_use] - pub fn markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::KeyValueXmlTags(KeyValueXmlTagsShape { - key_open: "".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - }), - } - } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::KeyValueXmlTagsShape; - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::Glm47KeyValueTagsFormat; - - #[test] - fn detects_glm47_template_with_arg_key_literal() { - let template = "{{- '' + tool_call.name }}{% for k, v in args.items() %}{{ k }}{{ v }}{% endfor %}"; - let markers = - Glm47KeyValueTagsFormat::detect(template).expect("GLM-4.7 template must be detected"); - - assert_eq!(markers.open, ""); - assert_eq!(markers.close, ""); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::KeyValueXmlTags(KeyValueXmlTagsShape { - key_open: "".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(Glm47KeyValueTagsFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(Glm47KeyValueTagsFormat::detect("").is_none()); - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs b/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs deleted file mode 100644 index a37c0fc4b..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/known_marker_candidates.rs +++ /dev/null @@ -1,45 +0,0 @@ -use llama_cpp_bindings_types::ToolCallMarkers; - -use crate::chat_template_tool_calls::tool_call_format_registration::ToolCallFormatRegistration; - -#[must_use] -pub fn known_marker_candidates() -> Vec { - ToolCallFormatRegistration::KNOWN - .iter() - .map(|registration| (registration.markers)()) - .collect() -} - -#[cfg(test)] -mod tests { - use std::collections::HashSet; - - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::known_marker_candidates; - use crate::chat_template_tool_calls::tool_call_format_registration::ToolCallFormatRegistration; - - #[test] - fn known_marker_candidates_returns_one_per_registered_shape() { - let candidates = known_marker_candidates(); - - assert_eq!(candidates.len(), ToolCallFormatRegistration::KNOWN.len()); - - let shape_discriminants: HashSet<&'static str> = candidates - .iter() - .map(|markers| match &markers.args_shape { - ToolCallArgsShape::BracketedJson(_) => "BracketedJson", - ToolCallArgsShape::JsonObject(_) => "JsonObject", - ToolCallArgsShape::KeyValueXmlTags(_) => "KeyValueXmlTags", - ToolCallArgsShape::PairedQuote(_) => "PairedQuote", - ToolCallArgsShape::XmlTags(_) => "XmlTags", - }) - .collect(); - - assert_eq!( - shape_discriminants.len(), - candidates.len(), - "duplicate shape discriminants in known_marker_candidates: {shape_discriminants:?}" - ); - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs b/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs deleted file mode 100644 index f930eb9ea..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/mistral3_arrow_args.rs +++ /dev/null @@ -1,68 +0,0 @@ -use llama_cpp_bindings_types::BracketedJsonShape; -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; - -pub struct Mistral3ArrowArgsFormat; - -impl Mistral3ArrowArgsFormat { - const TEMPLATE_FINGERPRINT: &'static str = "'[ARGS]'"; - - #[must_use] - pub fn markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "[TOOL_CALLS]".to_owned(), - close: String::new(), - args_shape: ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - } - } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::BracketedJsonShape; - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::Mistral3ArrowArgsFormat; - - #[test] - fn detects_mistral3_template_with_args_literal() { - let template = "...{{- name + '[ARGS]' + arguments }}..."; - let markers = - Mistral3ArrowArgsFormat::detect(template).expect("Mistral 3 template must be detected"); - - assert_eq!(markers.open, "[TOOL_CALLS]"); - assert_eq!(markers.close, ""); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(Mistral3ArrowArgsFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(Mistral3ArrowArgsFormat::detect("").is_none()); - } - - #[test] - fn returns_none_when_fingerprint_substring_appears_without_jinja_apostrophes() { - let template = "doc text mentioning the [ARGS] tag without quoting it as a literal"; - assert!(Mistral3ArrowArgsFormat::detect(template).is_none()); - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs b/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs deleted file mode 100644 index f3b799a29..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/qwen3_json_inside_tool_call.rs +++ /dev/null @@ -1,83 +0,0 @@ -use llama_cpp_bindings_types::JsonObjectShape; -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; - -pub struct Qwen3JsonInsideToolCallFormat; - -impl Qwen3JsonInsideToolCallFormat { - const TEMPLATE_FINGERPRINT_OPEN: &'static str = "'\\n{\"name\": \"'"; - const TEMPLATE_FINGERPRINT_ARGS_JOIN: &'static str = "'\", \"arguments\": '"; - - #[must_use] - pub fn markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::JsonObject(JsonObjectShape { - name_field: "name".to_owned(), - arguments_field: "arguments".to_owned(), - }), - } - } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT_OPEN) { - return None; - } - if !template.contains(Self::TEMPLATE_FINGERPRINT_ARGS_JOIN) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::JsonObjectShape; - use llama_cpp_bindings_types::ToolCallArgsShape; - - use super::Qwen3JsonInsideToolCallFormat; - - #[test] - fn detects_qwen3_json_inside_tool_call_template() { - let template = "{{- '\\n{\"name\": \"' + tool_call.name + '\", \"arguments\": ' + (tool_call.arguments | tojson) + '}\\n' -}}"; - let markers = Qwen3JsonInsideToolCallFormat::detect(template) - .expect("Qwen 3 template must be detected"); - - assert_eq!(markers.open, ""); - assert_eq!(markers.close, ""); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::JsonObject(JsonObjectShape { - name_field: "name".to_owned(), - arguments_field: "arguments".to_owned(), - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(Qwen3JsonInsideToolCallFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(Qwen3JsonInsideToolCallFormat::detect("").is_none()); - } - - #[test] - fn returns_none_when_only_open_fingerprint_present() { - let template = "{{- '\\n{\"name\": \"' + tool_call.name + ..."; - assert!( - Qwen3JsonInsideToolCallFormat::detect(template).is_none(), - "open fingerprint alone must not match (Qwen3-Embedding-style false positive)", - ); - } - - #[test] - fn returns_none_when_only_args_join_fingerprint_present() { - let template = "some text '\", \"arguments\": ' more text"; - assert!(Qwen3JsonInsideToolCallFormat::detect(template).is_none()); - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs b/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs deleted file mode 100644 index bb97c750e..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/qwen_xml_tags.rs +++ /dev/null @@ -1,74 +0,0 @@ -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; -use llama_cpp_bindings_types::XmlTagsShape; - -pub struct QwenXmlTagsFormat; - -impl QwenXmlTagsFormat { - const TEMPLATE_FINGERPRINT: &'static str = " ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::XmlTags(XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - }), - } - } - - #[must_use] - pub fn detect(template: &str) -> Option { - if !template.contains(Self::TEMPLATE_FINGERPRINT) { - return None; - } - Some(Self::markers()) - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::XmlTagsShape; - - use super::QwenXmlTagsFormat; - - #[test] - fn detects_qwen_xml_template_with_function_tag_literal() { - let template = "{{- '\\n\\n' }}"; - let markers = - QwenXmlTagsFormat::detect(template).expect("Qwen XML template must be detected"); - - assert_eq!(markers.open, ""); - assert_eq!(markers.close, ""); - assert_eq!( - markers.args_shape, - ToolCallArgsShape::XmlTags(XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - }) - ); - } - - #[test] - fn returns_none_for_template_without_fingerprint() { - assert!(QwenXmlTagsFormat::detect("just some plain template body").is_none()); - } - - #[test] - fn returns_none_for_empty_template() { - assert!(QwenXmlTagsFormat::detect("").is_none()); - } - - #[test] - fn detects_qwen_xml_template_with_concatenated_string_literal() { - let template = "{{- '\\n\\n\\n\\n' }}"; - assert!(QwenXmlTagsFormat::detect(template).is_some()); - } -} diff --git a/llama-cpp-bindings/src/chat_template_tool_calls/tool_call_format_registration.rs b/llama-cpp-bindings/src/chat_template_tool_calls/tool_call_format_registration.rs deleted file mode 100644 index c702b6d2e..000000000 --- a/llama-cpp-bindings/src/chat_template_tool_calls/tool_call_format_registration.rs +++ /dev/null @@ -1,40 +0,0 @@ -use llama_cpp_bindings_types::ToolCallMarkers; - -use crate::chat_template_tool_calls::gemma4_call_block::Gemma4CallBlockFormat; -use crate::chat_template_tool_calls::glm47_key_value_tags::Glm47KeyValueTagsFormat; -use crate::chat_template_tool_calls::mistral3_arrow_args::Mistral3ArrowArgsFormat; -use crate::chat_template_tool_calls::qwen_xml_tags::QwenXmlTagsFormat; -use crate::chat_template_tool_calls::qwen3_json_inside_tool_call::Qwen3JsonInsideToolCallFormat; - -pub struct ToolCallFormatRegistration { - pub detect: fn(&str) -> Option, - pub markers: fn() -> ToolCallMarkers, -} - -impl ToolCallFormatRegistration { - /// Every chat-template tool-call format this crate knows, ordered so that the more - /// restrictive shapes are tried before the ones whose separators could greedily - /// match them. - pub const KNOWN: &'static [Self] = &[ - Self { - detect: Qwen3JsonInsideToolCallFormat::detect, - markers: Qwen3JsonInsideToolCallFormat::markers, - }, - Self { - detect: QwenXmlTagsFormat::detect, - markers: QwenXmlTagsFormat::markers, - }, - Self { - detect: Glm47KeyValueTagsFormat::detect, - markers: Glm47KeyValueTagsFormat::markers, - }, - Self { - detect: Mistral3ArrowArgsFormat::detect, - markers: Mistral3ArrowArgsFormat::markers, - }, - Self { - detect: Gemma4CallBlockFormat::detect, - markers: Gemma4CallBlockFormat::markers, - }, - ]; -} diff --git a/llama-cpp-bindings/src/chat_templates.rs b/llama-cpp-bindings/src/chat_templates.rs new file mode 100644 index 000000000..10c23ae1d --- /dev/null +++ b/llama-cpp-bindings/src/chat_templates.rs @@ -0,0 +1,258 @@ +use std::ffi::CStr; +use std::ffi::c_char; +use std::ptr; +use std::ptr::NonNull; + +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + +use crate::error::chat_templates_error::ChatTemplatesError; +use crate::model::LlamaModel; +use crate::model::llama_chat_template::LlamaChatTemplate; + +fn chat_templates_init_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_chat_templates_init_status, + out_templates: llama_cpp_bindings_sys::llama_rs_chat_templates_handle, + out_error: *mut c_char, +) -> Result, ChatTemplatesError> { + match status { + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_OK => NonNull::new(out_templates) + .ok_or_else(|| { + FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "success status contained a null templates handle", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE => { + Err(ChatTemplatesError::ModelHasNoChatTemplate) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_ERROR_STRING_ALLOCATION_FAILED => { + Err(ChatTemplatesError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ChatTemplatesError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_chat_templates_init", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(ChatTemplatesError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG => { + Err(FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "was given a null model argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_TEMPLATE_OVERRIDE_ARG => { + Err(FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "was given a null template_override argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG => { + Err(FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "was given a null out_templates argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG => { + Err(FfiContractError { + operation: "llama_rs_chat_templates_init", + detail: "was given a null out_error argument", + } + .into()) + } + other => Err(FfiStatusError { + operation: "llama_rs_chat_templates_init", + code: i64::from(other), + } + .into()), + } +} + +/// Chat templates as llama.cpp prepares them for a model: with the model's BOS and EOS, the +/// template variant for tool calls, and llama.cpp's template fixes applied. +#[derive(Debug)] +pub struct ChatTemplates { + templates: NonNull, +} + +unsafe impl Send for ChatTemplates {} +unsafe impl Sync for ChatTemplates {} + +impl ChatTemplates { + fn init(model: &LlamaModel, template_override: &CStr) -> Result { + let mut out_templates: llama_cpp_bindings_sys::llama_rs_chat_templates_handle = + ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_chat_templates_init( + model.model.as_ptr(), + template_override.as_ptr(), + &raw mut out_templates, + &raw mut out_error, + ) + }; + + chat_templates_init_status_to_result(status, out_templates, out_error) + .map(|templates| Self { templates }) + } + + /// The model's own chat templates. + /// + /// # Errors + /// + /// Returns [`ChatTemplatesError::ModelHasNoChatTemplate`] for a model without a chat template, + /// or another [`ChatTemplatesError`] when llama.cpp cannot prepare the templates. + pub fn new(model: &LlamaModel) -> Result { + Self::init(model, c"") + } + + /// `template` in place of the model's own chat templates, the way llama.cpp serves a model + /// whose template cannot render a request, such as tool calls. + /// + /// # Errors + /// + /// Returns [`ChatTemplatesError::EmptyTemplateOverride`] for an empty `template`, or another + /// [`ChatTemplatesError`] when llama.cpp cannot prepare the template. + pub fn with_override( + model: &LlamaModel, + template: &LlamaChatTemplate, + ) -> Result { + if template.as_c_str().is_empty() { + return Err(ChatTemplatesError::EmptyTemplateOverride); + } + + Self::init(model, template.as_c_str()) + } + + #[must_use] + pub const fn as_ptr(&self) -> llama_cpp_bindings_sys::llama_rs_chat_templates_handle { + self.templates.as_ptr() + } +} + +impl Drop for ChatTemplates { + fn drop(&mut self) { + unsafe { llama_cpp_bindings_sys::llama_rs_chat_templates_free(self.templates.as_ptr()) }; + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::ffi::c_char; + use std::ptr; + + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::chat_templates_init_status_to_result; + use crate::error::chat_templates_error::ChatTemplatesError; + + fn init_error( + status: llama_cpp_bindings_sys::llama_rs_chat_templates_init_status, + out_error: *mut c_char, + ) -> Option { + chat_templates_init_status_to_result(status, ptr::null_mut(), out_error).err() + } + + const fn contract(detail: &'static str) -> ChatTemplatesError { + ChatTemplatesError::FfiContract(FfiContractError { + operation: "llama_rs_chat_templates_init", + detail, + }) + } + + fn missing_message(detail: &'static str) -> ChatTemplatesError { + ChatTemplatesError::CppString(CppStringError::Missing(FfiContractError { + operation: "llama_rs_chat_templates_init", + detail, + })) + } + + #[test] + fn preparing_templates_maps_every_failure_status() { + assert_eq!( + [ + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_OK, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_MODEL_HAS_NO_CHAT_TEMPLATE, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"the template threw".as_ptr()) } + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_MODEL_ARG, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_TEMPLATE_OVERRIDE_ARG, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_TEMPLATES_ARG, + ptr::null_mut() + ), + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_NULL_OUT_ERROR_ARG, + ptr::null_mut() + ), + init_error(255, ptr::null_mut()), + ], + [ + Some(contract("success status contained a null templates handle")), + Some(ChatTemplatesError::ModelHasNoChatTemplate), + Some(ChatTemplatesError::NotEnoughMemory), + Some(ChatTemplatesError::LlamaCppOutOfMemory), + Some(ChatTemplatesError::Reported { + message: "the template threw".to_owned() + }), + Some(contract("was given a null model argument")), + Some(contract("was given a null template_override argument")), + Some(contract("was given a null out_templates argument")), + Some(contract("was given a null out_error argument")), + Some(ChatTemplatesError::FfiStatus(FfiStatusError { + operation: "llama_rs_chat_templates_init", + code: 255, + })), + ] + ); + } + + #[test] + fn a_preparation_exception_without_a_message_is_a_contract_violation() { + assert_eq!( + init_error( + llama_cpp_bindings_sys::LLAMA_RS_CHAT_TEMPLATES_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut() + ), + Some(missing_message( + "reported a thrown C++ exception without an error message" + )) + ); + } +} diff --git a/llama-cpp-bindings/src/chat_tool_choice.rs b/llama-cpp-bindings/src/chat_tool_choice.rs new file mode 100644 index 000000000..fb09fad86 --- /dev/null +++ b/llama-cpp-bindings/src/chat_tool_choice.rs @@ -0,0 +1,16 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChatToolChoice { + Auto, + None, + Required, +} + +impl From for llama_cpp_bindings_sys::llama_rs_chat_tool_choice { + fn from(tool_choice: ChatToolChoice) -> Self { + match tool_choice { + ChatToolChoice::Auto => llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOL_CHOICE_AUTO, + ChatToolChoice::None => llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOL_CHOICE_NONE, + ChatToolChoice::Required => llama_cpp_bindings_sys::LLAMA_RS_CHAT_TOOL_CHOICE_REQUIRED, + } + } +} diff --git a/llama-cpp-bindings/src/context.rs b/llama-cpp-bindings/src/context.rs index ac4c4a223..6cb7ce090 100644 --- a/llama-cpp-bindings/src/context.rs +++ b/llama-cpp-bindings/src/context.rs @@ -1,5 +1,25 @@ +pub mod context_limits; +pub mod decoded_output_rows; +pub mod decoded_outputs; +pub mod kv_cache_type; +pub mod llama_attention_type; +pub mod llama_pooling_type; +pub mod llama_state_seq_flags; +pub mod load_seq_state_error; +pub mod load_session_error; +pub mod loaded_seq_state; +pub mod memory; +pub mod params; +pub mod rope_scaling_type; +pub mod save_seq_state_error; +pub mod save_session_error; +mod sequence_output_capacity; +pub mod session; +pub mod state_data_error; + use std::ffi::c_char; use std::ffi::c_void; +use std::fmt; use std::fmt::{Debug, Formatter}; use std::num::NonZeroI32; use std::ptr; @@ -9,23 +29,33 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use log::debug; + +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; +use crate::context::context_limits::ContextLimits; +use crate::context::decoded_output_rows::DecodedOutputRows; +use crate::context::decoded_outputs::DecodedOutputs; +use crate::context::llama_pooling_type::LlamaPoolingType; use crate::context::params::LlamaContextParams; -use crate::context::params::LlamaPoolingType; use crate::context::sequence_output_capacity::SequenceOutputCapacity; +use crate::error::batch_transfer_error::BatchTransferError; +use crate::error::decode_error::DecodeError; +use crate::error::embeddings_error::EmbeddingsError; +use crate::error::encode_error::EncodeError; +use crate::error::llama_context_load_error::LlamaContextLoadError; +use crate::error::llama_lora_adapters_error::LlamaLoraAdaptersError; +use crate::error::logits_error::LogitsError; +use crate::error::process_failure::ProcessFailure; use crate::llama_backend::LlamaBackend; use crate::llama_batch::LlamaBatch; use crate::model::LlamaModel; use crate::model::lora_adapter_scale::LoraAdapterScale; -use crate::timing::LlamaTimings; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; use crate::token::data_array::LlamaTokenDataArray; -use crate::{ - DecodeError, EmbeddingsError, EncodeError, LlamaContextLoadError, LlamaLoraAdaptersError, - LogitsError, -}; const fn check_lora_adapters_result(err_code: i32) -> Result<(), LlamaLoraAdaptersError> { if err_code != 0 { @@ -35,63 +65,6 @@ const fn check_lora_adapters_result(err_code: i32) -> Result<(), LlamaLoraAdapte Ok(()) } -fn nextn_hidden_state_probe_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_context_nextn_hidden_state_probe_status, - produces_nextn_hidden_state: bool, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK => { - Ok(produces_nextn_hidden_state) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null ctx argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null out_produces_nextn_hidden_state argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null out_error argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_FAILED_TO_RESERVE_GRAPH => { - Err(EmbeddingsError::NextnProbeGraphReservationFailed) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_ERROR_STRING_ALLOCATION_FAILED => { - Err(EmbeddingsError::NextnProbeNotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(EmbeddingsError::NextnProbeLlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_context_graph_produces_nextn_hidden_state", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(EmbeddingsError::NextnProbeReported { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - code: i64::from(other), - } - .into()), - } -} - fn new_context_with_model_status_to_result( status: llama_cpp_bindings_sys::llama_rs_new_context_with_model_status, out_ctx: *mut llama_cpp_bindings_sys::llama_context, @@ -100,7 +73,7 @@ fn new_context_with_model_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_OK => NonNull::new(out_ctx) .ok_or_else(|| { - crate::FfiContractError { + FfiContractError { operation: "llama_rs_new_context_with_model", detail: "success status contained a null context", } @@ -126,27 +99,27 @@ fn new_context_with_model_status_to_result( Err(LlamaContextLoadError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_MODEL_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_CTX_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_new_context_with_model", code: i64::from(other), } @@ -154,162 +127,145 @@ fn new_context_with_model_status_to_result( } } -fn decode_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_decode_status, - out_llama_cpp_return_code: i32, +fn batch_ext_init_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_batch_ext_init_status, + out_batch: *mut llama_cpp_bindings_sys::llama_batch_ext, out_error: *mut c_char, -) -> Result<(), DecodeError> { +) -> Result, LlamaContextLoadError> { match status { - llama_cpp_bindings_sys::LLAMA_RS_DECODE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE => { - let code = - NonZeroI32::new(out_llama_cpp_return_code).ok_or(crate::FfiContractError { - operation: "llama_rs_decode", - detail: "nonzero llama.cpp return status contained zero", - })?; - Err(DecodeError::from(code)) - } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_OUT_OF_MEMORY => { - Err(DecodeError::DecodeOutOfMemory) + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_OK => { + NonNull::new(out_batch).ok_or_else(|| { + FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "success status contained a null batch", + } + .into() + }) } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_COMPUTE_FAILED => Err(DecodeError::ComputeFailed), - llama_cpp_bindings_sys::LLAMA_RS_DECODE_ERROR_STRING_ALLOCATION_FAILED => { - Err(DecodeError::NotEnoughMemory) + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_ERROR_STRING_ALLOCATION_FAILED => { + Err(LlamaContextLoadError::NotEnoughMemory) } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(DecodeError::LlamaCppOutOfMemory) + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_OUT_OF_MEMORY => { + Err(LlamaContextLoadError::LlamaCppOutOfMemory) } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { read_and_free_cpp_string( out_error, - "llama_rs_decode", + "llama_rs_batch_ext_init", "reported a thrown C++ exception without an error message", ) }?; - Err(DecodeError::Reported { message }) + Err(LlamaContextLoadError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_CTX_ARG => Err(crate::FfiContractError { - operation: "llama_rs_decode", + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_batch_ext_init", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_decode", + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG => { + Err(FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "was given a null out_batch argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG => { + Err(FfiContractError { + operation: "llama_rs_batch_ext_init", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { - operation: "llama_rs_decode", + other => Err(FfiStatusError { + operation: "llama_rs_batch_ext_init", code: i64::from(other), } .into()), } } -fn encode_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_encode_status, +fn process_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_process_status, out_llama_cpp_return_code: i32, out_error: *mut c_char, -) -> Result<(), EncodeError> { +) -> Result<(), TProcessError> +where + TProcessError: From + From, +{ + const OPERATION: &str = "llama_rs_process"; + + let contract_violation = |detail: &'static str| { + ProcessFailure::from(FfiContractError { + operation: OPERATION, + detail, + }) + }; + match status { - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER => { - Err(EncodeError::ModelHasNoEncoder) - } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE => { - let code = - NonZeroI32::new(out_llama_cpp_return_code).ok_or(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "nonzero llama.cpp return status contained zero", - })?; - Err(EncodeError::from(code)) - } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_OUT_OF_MEMORY => { - Err(EncodeError::EncodeOutOfMemory) + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE => { + let code = NonZeroI32::new(out_llama_cpp_return_code).ok_or_else(|| { + contract_violation("nonzero llama.cpp return status contained zero") + })?; + + Err(TProcessError::from(code)) } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_COMPUTE_FAILED => Err(EncodeError::ComputeFailed), - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_ERROR_STRING_ALLOCATION_FAILED => { - Err(EncodeError::NotEnoughMemory) + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_ERROR_STRING_ALLOCATION_FAILED => { + Err(ProcessFailure::NotEnoughMemory.into()) } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(EncodeError::LlamaCppOutOfMemory) + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_OUT_OF_MEMORY => { + Err(ProcessFailure::LlamaCppOutOfMemory.into()) } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { read_and_free_cpp_string( out_error, - "llama_rs_encode", + OPERATION, "reported a thrown C++ exception without an error message", ) - }?; - Err(EncodeError::Reported { message }) + } + .map_err(ProcessFailure::from)?; + + Err(ProcessFailure::Reported { message }.into()) } - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_CTX_ARG => Err(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null ctx argument", + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_CTX_ARG => { + Err(contract_violation("was given a null ctx argument").into()) } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_MODEL => Err(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null model argument", + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_BATCH_ARG => { + Err(contract_violation("was given a null batch argument").into()) } - .into()), - other => Err(crate::FfiStatusError { - operation: "llama_rs_encode", - code: i64::from(other), + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG => { + Err(contract_violation("was given a null out_llama_cpp_return_code argument").into()) + } + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG => { + Err(contract_violation("was given a null out_error argument").into()) } + other => Err(ProcessFailure::from(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + }) .into()), } } -fn token_index_within_context(token_index: i32, context_size: u32) -> Result<(), LogitsError> { - if token_index >= 0 { - let token_index_u32 = - u32::try_from(token_index).map_err(LogitsError::TokenIndexOverflow)?; +fn memory_of( + context: NonNull, +) -> Option { + let memory = unsafe { llama_cpp_bindings_sys::llama_get_memory(context.as_ptr()) }; - if context_size <= token_index_u32 { - return Err(LogitsError::TokenIndexExceedsContext { - token_index: token_index_u32, - context_size, - }); - } - } - - Ok(()) + (!memory.is_null()).then_some(memory) } -unsafe fn logits_slice_from_raw_parts<'logits>( - data: *const f32, +unsafe fn logits_slice<'logits>( + logits: NonNull, n_vocab: i32, ) -> Result<&'logits [f32], LogitsError> { - if data.is_null() { - return Err(LogitsError::NullLogits); - } - let len = usize::try_from(n_vocab).map_err(LogitsError::VocabSizeOverflow)?; - Ok(unsafe { slice::from_raw_parts(data, len) }) + Ok(unsafe { slice::from_raw_parts(logits.as_ptr(), len) }) } -pub mod kv_cache; -pub mod kv_cache_type; -pub mod llama_attention_type; -pub mod llama_pooling_type; -pub mod llama_state_seq_flags; -pub mod load_seq_state_error; -pub mod load_session_error; -pub mod loaded_seq_state; -pub mod params; -pub mod rope_scaling_type; -pub mod save_seq_state_error; -pub mod save_session_error; -mod sequence_output_capacity; -pub mod session; -pub mod state_data_error; - unsafe extern "C" fn abort_callback_trampoline(data: *mut c_void) -> bool { let flag = unsafe { &*(data as *const AtomicBool) }; @@ -320,11 +276,15 @@ pub struct LlamaContext<'model> { pub context: NonNull, pub model: &'model LlamaModel, abort_flag: Option>, - initialized_logits: Vec, + decoded_outputs: DecodedOutputs, + batch_ext: NonNull, + limits: ContextLimits, + keeps_no_memory: bool, + embeddings: bool, } impl Debug for LlamaContext<'_> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("LlamaContext") .field("context", &self.context) .finish() @@ -332,17 +292,51 @@ impl Debug for LlamaContext<'_> { } impl<'model> LlamaContext<'model> { - #[must_use] - pub const fn new( - llama_model: &'model LlamaModel, - llama_context: NonNull, - ) -> Self { - Self { - context: llama_context, - model: llama_model, + fn with_batch( + model: &'model LlamaModel, + context: NonNull, + embeddings: bool, + ) -> Result { + let pooling_type = LlamaPoolingType::try_from(unsafe { + llama_cpp_bindings_sys::llama_rs_pooling_type(context.as_ptr()) + }) + .inspect_err(|_unknown_pooling_type| unsafe { + llama_cpp_bindings_sys::llama_free(context.as_ptr()); + })?; + let mut out_batch: *mut llama_cpp_bindings_sys::llama_batch_ext = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_batch_ext_init( + context.as_ptr(), + &raw mut out_batch, + &raw mut out_error, + ) + }; + let batch_ext = batch_ext_init_status_to_result(status, out_batch, out_error).inspect_err( + |_batch_init_failure| unsafe { + llama_cpp_bindings_sys::llama_free(context.as_ptr()); + }, + )?; + let keeps_no_memory = memory_of(context).is_none(); + + Ok(Self { + context, + model, abort_flag: None, - initialized_logits: Vec::new(), - } + decoded_outputs: DecodedOutputs { + rows: DecodedOutputRows::BatchIndices(Vec::new()), + pooling_type, + }, + batch_ext, + limits: ContextLimits { + n_batch: unsafe { llama_cpp_bindings_sys::llama_n_batch(context.as_ptr()) }, + n_ubatch: unsafe { llama_cpp_bindings_sys::llama_n_ubatch(context.as_ptr()) }, + n_seq_max: unsafe { llama_cpp_bindings_sys::llama_n_seq_max(context.as_ptr()) }, + n_vocab: model.n_vocab(), + }, + keeps_no_memory, + embeddings, + }) } /// # Errors @@ -355,15 +349,14 @@ impl<'model> LlamaContext<'model> { params: LlamaContextParams, ) -> Result { let context_params = params.context_params; + let embeddings = params.embeddings(); SequenceOutputCapacity { - attention_type: params.attention_type(), + attention_type: params.attention_type()?, model_causal_attention: unsafe { llama_cpp_bindings_sys::llama_rs_model_causal_attn(model.model.as_ptr()) }, - model_has_encoder: unsafe { - llama_cpp_bindings_sys::llama_model_has_encoder(model.model.as_ptr()) - }, + model_has_encoder: model.has_encoder(), model_n_ctx_train: unsafe { llama_cpp_bindings_sys::llama_model_n_ctx_train(model.model.as_ptr()) } @@ -387,17 +380,27 @@ impl<'model> LlamaContext<'model> { }; let context = new_context_with_model_status_to_result(status, out_ctx, out_error)?; - Ok(Self::new(model, context)) + Self::with_batch(model, context, embeddings) + } + + #[must_use] + pub const fn n_batch(&self) -> u32 { + self.limits.n_batch } #[must_use] - pub fn n_batch(&self) -> u32 { - unsafe { llama_cpp_bindings_sys::llama_n_batch(self.context.as_ptr()) } + pub const fn n_ubatch(&self) -> u32 { + self.limits.n_ubatch } #[must_use] - pub fn n_ubatch(&self) -> u32 { - unsafe { llama_cpp_bindings_sys::llama_n_ubatch(self.context.as_ptr()) } + pub const fn n_seq_max(&self) -> u32 { + self.limits.n_seq_max + } + + fn decodes_batches_in_one_micro_batch(&self) -> bool { + self.keeps_no_memory + || !unsafe { llama_cpp_bindings_sys::llama_get_causal_attn(self.context.as_ptr()) } } fn exceeds_micro_batch(&self, batch: &LlamaBatch) -> bool { @@ -414,7 +417,6 @@ impl<'model> LlamaContext<'model> { unsafe { llama_cpp_bindings_sys::llama_n_ctx_seq(self.context.as_ptr()) } } - #[expect(unsafe_code, reason = "required for FFI abort callback registration")] pub fn set_abort_flag(&mut self, flag: Arc) { let raw_ptr = Arc::as_ptr(&flag) as *mut c_void; self.abort_flag = Some(flag); @@ -428,7 +430,6 @@ impl<'model> LlamaContext<'model> { } } - #[expect(unsafe_code, reason = "required for FFI abort callback deregistration")] pub fn clear_abort_callback(&mut self) { self.abort_flag = None; @@ -441,59 +442,80 @@ impl<'model> LlamaContext<'model> { } } - #[expect(unsafe_code, reason = "required for FFI synchronization call")] - pub fn synchronize(&self) { - unsafe { llama_cpp_bindings_sys::llama_synchronize(self.context.as_ptr()) } - } - - #[expect(unsafe_code, reason = "required for FFI threadpool detachment")] - pub fn detach_threadpool(&self) { - unsafe { llama_cpp_bindings_sys::llama_detach_threadpool(self.context.as_ptr()) } + /// Records the rows a llama.cpp helper wrote when it decoded on this context without one of + /// this context's batches, such as the mtmd helper decoding media chunks. + pub fn record_helper_decode(&mut self, logits_last: bool) { + self.decoded_outputs.rows = DecodedOutputRows::of_helper_decode(logits_last); } - pub fn mark_logits_initialized(&mut self, token_index: i32) { - self.initialized_logits = vec![token_index]; + fn record_batch_output_rows(&mut self, output_token_indices: &[i32]) { + if let DecodedOutputRows::BatchIndices(rows) = &mut self.decoded_outputs.rows { + rows.clear(); + rows.extend_from_slice(output_token_indices); + } else { + self.decoded_outputs.rows = + DecodedOutputRows::BatchIndices(output_token_indices.to_vec()); + } } - /// # Errors - /// - /// - `DecodeError` if the decoding failed. - pub fn decode(&mut self, batch: &mut LlamaBatch) -> Result<(), DecodeError> { - let decodes_batches_in_one_micro_batch = unsafe { - llama_cpp_bindings_sys::llama_rs_context_decodes_batches_in_one_micro_batch( - self.context.as_ptr(), - ) - }; - - if decodes_batches_in_one_micro_batch && self.exceeds_micro_batch(batch) { - return Err(DecodeError::BatchExceedsMicroBatch { - n_tokens: batch.n_tokens(), - n_ubatch: self.n_ubatch(), + fn transfer_batch(&self, batch: &LlamaBatch) -> Result<(), BatchTransferError> { + if batch.limits() != self.limits { + return Err(BatchTransferError::LimitsMismatch { + batch: batch.limits(), + context: self.limits, }); } + unsafe { batch.assign_to(self.batch_ext) } + } + + fn process( + &mut self, + process_type: llama_cpp_bindings_sys::llama_process_type, + batch: &LlamaBatch, + ) -> Result<(), TProcessError> + where + TProcessError: From + From + From, + { + self.transfer_batch(batch)?; + let mut out_llama_cpp_return_code: i32 = 0; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { - llama_cpp_bindings_sys::llama_rs_decode( + llama_cpp_bindings_sys::llama_rs_process( self.context.as_ptr(), - batch.llama_batch, + process_type, + self.batch_ext.as_ptr(), &raw mut out_llama_cpp_return_code, &raw mut out_error, ) }; - decode_status_to_result(status, out_llama_cpp_return_code, out_error)?; - self.initialized_logits - .clone_from(&batch.initialized_logits); + process_status_to_result::(status, out_llama_cpp_return_code, out_error)?; + self.record_batch_output_rows(batch.output_token_indices()); Ok(()) } /// # Errors /// - /// - `EncodeError` if the encoding failed. - pub fn encode(&mut self, batch: &mut LlamaBatch) -> Result<(), EncodeError> { + /// - `DecodeError` if the decoding failed. + pub fn decode(&mut self, batch: &LlamaBatch) -> Result<(), DecodeError> { + if self.decodes_batches_in_one_micro_batch() && self.exceeds_micro_batch(batch) { + return Err(DecodeError::BatchExceedsMicroBatch { + n_tokens: batch.n_tokens(), + n_ubatch: self.n_ubatch(), + }); + } + + self.process(llama_cpp_bindings_sys::LLAMA_PROCESS_TYPE_DECODE, batch) + } + + /// # Errors + /// + /// - `EncodeError` if the encoding failed, or [`EncodeError::ModelHasNoEncoder`] when the + /// model has no encoder to run. + pub fn encode(&mut self, batch: &LlamaBatch) -> Result<(), EncodeError> { if self.exceeds_micro_batch(batch) { return Err(EncodeError::BatchExceedsMicroBatch { n_tokens: batch.n_tokens(), @@ -501,52 +523,27 @@ impl<'model> LlamaContext<'model> { }); } - let mut out_llama_cpp_return_code: i32 = 0; - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_encode( - self.context.as_ptr(), - batch.llama_batch, - &raw mut out_llama_cpp_return_code, - &raw mut out_error, - ) - }; - encode_status_to_result(status, out_llama_cpp_return_code, out_error)?; - - self.initialized_logits - .clone_from(&batch.initialized_logits); + if !self.model.has_encoder() { + return Err(EncodeError::ModelHasNoEncoder); + } - Ok(()) + self.process(llama_cpp_bindings_sys::LLAMA_PROCESS_TYPE_ENCODE, batch) } #[must_use] - pub fn pooling_type(&self) -> LlamaPoolingType { - LlamaPoolingType::from(unsafe { - llama_cpp_bindings_sys::llama_pooling_type(self.context.as_ptr()) - }) + pub const fn pooling_type(&self) -> LlamaPoolingType { + self.decoded_outputs.pooling_type } pub fn set_embeddings(&mut self, embeddings: bool) { unsafe { llama_cpp_bindings_sys::llama_set_embeddings(self.context.as_ptr(), embeddings); } + self.embeddings = embeddings; } - fn embeddings_enabled(&self) -> bool { - unsafe { - llama_cpp_bindings_sys::llama_rs_context_embeddings_enabled(self.context.as_ptr()) - } - } - - fn masked_nextn_embeddings_enabled(&self) -> bool { - let nextn_enabled = unsafe { - llama_cpp_bindings_sys::llama_rs_context_embeddings_nextn_enabled(self.context.as_ptr()) - }; - let nextn_masked = unsafe { - llama_cpp_bindings_sys::llama_rs_context_embeddings_nextn_masked(self.context.as_ptr()) - }; - - nextn_enabled && nextn_masked + const fn embeddings_enabled(&self) -> bool { + self.embeddings } fn output_embedding_width(&self) -> Result { @@ -600,8 +597,8 @@ impl<'model> LlamaContext<'model> { /// - When embeddings are not enabled on this context. /// - When the context pools its outputs, because llama.cpp then writes only sequence /// embeddings and never the token rows. - /// - When the given token didn't have logits enabled when it was passed. - /// - If the given token index exceeds the max token id. + /// - When `token_index` names no output row of the last decode. With embeddings enabled, + /// llama.cpp outputs every token of the batch. pub fn embeddings_ith(&self, token_index: i32) -> Result<&[f32], EmbeddingsError> { if !self.embeddings_enabled() { return Err(EmbeddingsError::NotEnabled); @@ -618,114 +615,22 @@ impl<'model> LlamaContext<'model> { }; if embedding.is_null() { - Err(EmbeddingsError::LogitsNotEnabled) + Err(EmbeddingsError::TokenNotAnOutput { token_index }) } else { self.output_embedding_row(embedding) } } - /// # Errors - /// - /// - When the context pools its outputs, because llama.cpp only extracts `NextN` embeddings - /// with `LLAMA_POOLING_TYPE_NONE`. - /// - When the model's graph produces no `NextN` hidden state; `NextN` embeddings stay disabled. - /// - When llama.cpp fails while checking the graph; `NextN` embeddings stay disabled. - pub fn enable_masked_nextn_embeddings(&mut self) -> Result<(), EmbeddingsError> { - let pooling_type = self.pooling_type(); - - if pooling_type != LlamaPoolingType::None { - return Err(EmbeddingsError::NextnEmbeddingsRequireNonePooling { pooling_type }); - } - - self.set_masked_nextn_embeddings(true); - - let enabled = - self.graph_produces_nextn_hidden_state() - .and_then(|produces_nextn_hidden_state| { - produces_nextn_hidden_state - .then_some(()) - .ok_or(EmbeddingsError::NextnEmbeddingsUnsupported) - }); - - if enabled.is_err() { - self.set_masked_nextn_embeddings(false); - } - - enabled - } - - fn set_masked_nextn_embeddings(&mut self, enabled: bool) { - unsafe { - llama_cpp_bindings_sys::llama_rs_set_embeddings_nextn( - self.context.as_ptr(), - enabled, - enabled, - ); - } - } - - fn graph_produces_nextn_hidden_state(&self) -> Result { - let mut produces_nextn_hidden_state = false; - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_context_graph_produces_nextn_hidden_state( - self.context.as_ptr(), - &raw mut produces_nextn_hidden_state, - &raw mut out_error, - ) - }; - - nextn_hidden_state_probe_status_to_result(status, produces_nextn_hidden_state, out_error) - } - - /// # Errors - /// - /// - When masked `NextN` embeddings were not enabled on this context. - /// - When the given token was not marked as an output of the last decoded batch. - pub fn nextn_embeddings_ith(&self, token_index: i32) -> Result<&[f32], EmbeddingsError> { - if !self.masked_nextn_embeddings_enabled() { - return Err(EmbeddingsError::NextnEmbeddingsNotEnabled); - } - - let embedding = unsafe { - llama_cpp_bindings_sys::llama_rs_get_embeddings_nextn_ith( - self.context.as_ptr(), - token_index, - ) - }; - - if embedding.is_null() { - Err(EmbeddingsError::NextnEmbeddingUnavailable { token_index }) + fn ensure_logits_produced(&self) -> Result<(), LogitsError> { + if unsafe { + llama_cpp_bindings_sys::llama_rs_context_produced_logits(self.context.as_ptr()) + } { + Ok(()) } else { - self.output_embedding_row(embedding) + Err(self.decoded_outputs.missing_logits_error()) } } - /// # Errors - /// Returns `LogitsError` if logits are null or `n_vocab` overflows. - pub fn candidates(&self) -> Result + '_, LogitsError> { - let logits = self.get_logits()?; - - Ok((0_i32..).zip(logits).map(|(token_id, logit)| { - let token = LlamaToken::new(token_id); - LlamaTokenData::new(token, *logit, 0_f32) - })) - } - - /// # Errors - /// Returns `LogitsError` if logits are null or `n_vocab` overflows. - pub fn token_data_array(&self) -> Result { - Ok(LlamaTokenDataArray::from_iter(self.candidates()?, false)) - } - - /// # Errors - /// Returns `LogitsError` if the logits pointer is null or `n_vocab` overflows. - pub fn get_logits(&self) -> Result<&[f32], LogitsError> { - let data = unsafe { llama_cpp_bindings_sys::llama_get_logits(self.context.as_ptr()) }; - - unsafe { logits_slice_from_raw_parts(data, self.model.n_vocab()) } - } - /// # Errors /// Returns `LogitsError` if the token is not initialized or out of range. pub fn candidates_ith( @@ -768,29 +673,22 @@ impl<'model> LlamaContext<'model> { Ok(()) } + /// Accepts the batch index of an output token, or a negative index counting back from the + /// last output. + /// /// # Errors - /// Returns `LogitsError` if the token is not initialized, out of range, or `n_vocab` overflows. + /// Returns `LogitsError` if the token is not an output of the last decode, out of range, the + /// last decode produced no logits, or `n_vocab` overflows. pub fn get_logits_ith(&self, token_index: i32) -> Result<&[f32], LogitsError> { - if !self.initialized_logits.contains(&token_index) { - return Err(LogitsError::TokenNotInitialized(token_index)); - } - - token_index_within_context(token_index, self.n_ctx())?; - - let data = unsafe { - llama_cpp_bindings_sys::llama_get_logits_ith(self.context.as_ptr(), token_index) - }; - - unsafe { logits_slice_from_raw_parts(data, self.model.n_vocab()) } - } + self.decoded_outputs.ensure_output(token_index)?; + self.ensure_logits_produced()?; - pub fn reset_timings(&mut self) { - unsafe { llama_cpp_bindings_sys::llama_perf_context_reset(self.context.as_ptr()) } - } + let row = NonNull::new(unsafe { + llama_cpp_bindings_sys::llama_rs_get_logits_ith(self.context.as_ptr(), token_index) + }) + .ok_or(LogitsError::NullLogits)?; - pub fn timings(&mut self) -> LlamaTimings { - let timings = unsafe { llama_cpp_bindings_sys::llama_perf_context(self.context.as_ptr()) }; - LlamaTimings { timings } + unsafe { logits_slice(row, self.model.n_vocab()) } } /// # Errors @@ -813,7 +711,7 @@ impl<'model> LlamaContext<'model> { .map_or(ptr::null_mut(), ptr::from_mut); let scales_ptr = scales.first_mut().map_or(ptr::null_mut(), ptr::from_mut); let err_code = unsafe { - llama_cpp_bindings_sys::llama_set_adapters_lora( + llama_cpp_bindings_sys::llama_rs_set_adapters_lora( self.context.as_ptr(), raw_adapters_ptr, raw_adapters.len(), @@ -822,29 +720,32 @@ impl<'model> LlamaContext<'model> { }; check_lora_adapters_result(err_code)?; - log::debug!("Updated lora adapters"); + debug!("Updated lora adapters"); Ok(()) } } impl Drop for LlamaContext<'_> { fn drop(&mut self) { - unsafe { llama_cpp_bindings_sys::llama_free(self.context.as_ptr()) } + unsafe { + llama_cpp_bindings_sys::llama_batch_ext_free(self.batch_ext.as_ptr()); + llama_cpp_bindings_sys::llama_free(self.context.as_ptr()); + } } } #[cfg(test)] mod check_lora_adapters_result_tests { use super::check_lora_adapters_result; - use crate::LlamaLoraAdaptersError; + use crate::error::llama_lora_adapters_error::LlamaLoraAdaptersError; #[test] - fn check_lora_adapters_result_ok_for_zero() { - assert!(check_lora_adapters_result(0).is_ok()); + fn a_zero_set_adapters_return_code_is_success() { + assert_eq!(check_lora_adapters_result(0), Ok(())); } #[test] - fn check_lora_adapters_result_error_for_nonzero() { + fn a_nonzero_set_adapters_return_code_is_an_error() { let result = check_lora_adapters_result(-1); assert_eq!(result, Err(LlamaLoraAdaptersError::ErrorResult(-1))); @@ -852,210 +753,72 @@ mod check_lora_adapters_result_tests { } #[cfg(test)] -mod nextn_hidden_state_probe_status_to_result_tests { +mod new_context_with_model_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; - use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; - use super::nextn_hidden_state_probe_status_to_result; - use crate::EmbeddingsError; + use super::new_context_with_model_status_to_result; + use crate::error::llama_context_load_error::LlamaContextLoadError; #[test] - fn nextn_hidden_state_probe_status_to_result_maps_ok_to_whether_the_graph_produces_it() { + fn new_context_success_with_null_context_is_contract_error() { + let result = new_context_with_model_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_OK, + ptr::null_mut(), + ptr::null_mut(), + ); + assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_OK, - true, - ptr::null_mut(), - ), - Ok(true) + result, + Err(LlamaContextLoadError::FfiContract(FfiContractError { + operation: "llama_rs_new_context_with_model", + detail: "success status contained a null context", + })) ); } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_null_ctx_arg() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_CTX_ARG, - false, - ptr::null_mut(), - ), - Err(crate::FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null ctx argument", - } - .into()) + fn new_context_llama_cpp_returned_null_maps_unconstructible() { + let result = new_context_with_model_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_LLAMA_CPP_RETURNED_NULL, + ptr::null_mut(), + ptr::null_mut(), ); + + assert_eq!(result, Err(LlamaContextLoadError::Unconstructible)); } #[test] - fn nextn_hidden_state_probe_status_to_result_maps_null_out_produces_nextn_hidden_state_arg() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_PRODUCES_NEXTN_HIDDEN_STATE_ARG, - false, - ptr::null_mut(), - ), - Err(crate::FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null out_produces_nextn_hidden_state argument", - } - .into()) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_maps_null_out_error_arg() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_NULL_OUT_ERROR_ARG, - false, - ptr::null_mut(), - ), - Err(crate::FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "was given a null out_error argument", - } - .into()) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_maps_llama_cpp_failed_to_reserve_graph() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_FAILED_TO_RESERVE_GRAPH, - false, - ptr::null_mut(), - ), - Err(EmbeddingsError::NextnProbeGraphReservationFailed) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_maps_error_string_allocation_failed() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_ERROR_STRING_ALLOCATION_FAILED, - false, - ptr::null_mut(), - ), - Err(EmbeddingsError::NextnProbeNotEnoughMemory) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_maps_llama_cpp_out_of_memory() { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_OUT_OF_MEMORY, - false, - ptr::null_mut(), - ), - Err(EmbeddingsError::NextnProbeLlamaCppOutOfMemory) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_maps_a_thrown_exception_with_its_message() { - let expected_message = unsafe { - read_and_free_cpp_string( - wrapper_allocated_error_message_pointer(), - "llama_rs_context_graph_produces_nextn_hidden_state", - "reported a thrown C++ exception without an error message", - ) - }; - - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION, - false, - wrapper_allocated_error_message_pointer(), - ) - .err(), - expected_message - .ok() - .map(|message| EmbeddingsError::NextnProbeReported { message }) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_maps_llama_cpp_threw_cxx_exception_without_a_message() - { - assert_eq!( - nextn_hidden_state_probe_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CONTEXT_NEXTN_HIDDEN_STATE_PROBE_LLAMA_CPP_THREW_CXX_EXCEPTION, - false, - ptr::null_mut(), - ), - Err(crate::FfiContractError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn nextn_hidden_state_probe_status_to_result_maps_an_unknown_status() { - assert_eq!( - nextn_hidden_state_probe_status_to_result(255, false, ptr::null_mut()), - Err(crate::FfiStatusError { - operation: "llama_rs_context_graph_produces_nextn_hidden_state", - code: 255, - } - .into()) - ); - } -} - -#[cfg(test)] -mod new_context_with_model_status_to_result_tests { - use std::ptr; - - use super::new_context_with_model_status_to_result; - use crate::LlamaContextLoadError; - - #[test] - fn new_context_success_with_null_context_is_contract_error() { + fn new_context_allocation_failed_maps_not_enough_memory() { let result = new_context_with_model_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_OK, + llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_ERROR_STRING_ALLOCATION_FAILED, ptr::null_mut(), ptr::null_mut(), ); - assert_eq!( - result, - Err(LlamaContextLoadError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_new_context_with_model", - detail: "success status contained a null context", - } - )) - ); + assert_eq!(result, Err(LlamaContextLoadError::NotEnoughMemory)); } #[test] - fn new_context_llama_cpp_returned_null_maps_unconstructible() { + fn new_context_cxx_exception_reports_its_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"context construction threw".as_ptr()) + }; let result = new_context_with_model_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_LLAMA_CPP_RETURNED_NULL, - ptr::null_mut(), + llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), + out_error, ); - assert_eq!(result, Err(LlamaContextLoadError::Unconstructible)); - } - - #[test] - fn new_context_allocation_failed_maps_not_enough_memory() { - let result = new_context_with_model_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ptr::null_mut(), + assert_eq!( + result, + Err(LlamaContextLoadError::Reported { + message: "context construction threw".to_owned() + }) ); - - assert_eq!(result, Err(LlamaContextLoadError::NotEnoughMemory)); } #[test] @@ -1068,10 +831,10 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_new_context_with_model", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -1082,7 +845,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( result, - Err(LlamaContextLoadError::FfiStatus(crate::FfiStatusError { + Err(LlamaContextLoadError::FfiStatus(FfiStatusError { operation: "llama_rs_new_context_with_model", code: 255, })) @@ -1090,7 +853,7 @@ mod new_context_with_model_status_to_result_tests { } #[test] - fn new_context_with_model_status_to_result_maps_null_model_arg() { + fn new_context_with_model_status_to_result_reports_a_null_model_arg_as_a_contract_violation() { let outcome = new_context_with_model_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_MODEL_ARG, ptr::null_mut(), @@ -1099,7 +862,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null model argument", } @@ -1109,7 +872,8 @@ mod new_context_with_model_status_to_result_tests { } #[test] - fn new_context_with_model_status_to_result_maps_null_out_ctx_arg() { + fn new_context_with_model_status_to_result_reports_a_null_out_ctx_arg_as_a_contract_violation() + { let outcome = new_context_with_model_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_CTX_ARG, ptr::null_mut(), @@ -1118,7 +882,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_ctx argument", } @@ -1128,7 +892,8 @@ mod new_context_with_model_status_to_result_tests { } #[test] - fn new_context_with_model_status_to_result_maps_null_out_error_arg() { + fn new_context_with_model_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() + { let outcome = new_context_with_model_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_NULL_OUT_ERROR_ARG, ptr::null_mut(), @@ -1137,7 +902,7 @@ mod new_context_with_model_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_new_context_with_model", detail: "was given a null out_error argument", } @@ -1147,7 +912,7 @@ mod new_context_with_model_status_to_result_tests { } #[test] - fn new_context_with_model_status_to_result_maps_llama_cpp_out_of_memory() { + fn new_context_with_model_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = new_context_with_model_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_NEW_CONTEXT_WITH_MODEL_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), @@ -1161,359 +926,249 @@ mod new_context_with_model_status_to_result_tests { } #[cfg(test)] -mod decode_status_to_result_tests { +mod batch_ext_init_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; - use super::decode_status_to_result; - use crate::DecodeError; - - #[test] - fn decode_nonzero_code_maps_from_code() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 1, - ptr::null_mut(), - ); - - assert_eq!(result, Err(DecodeError::NoKvCacheSlot)); - } - - #[test] - fn decode_out_of_memory_maps_decode_out_of_memory() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(DecodeError::DecodeOutOfMemory)); - } + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; - #[test] - fn decode_compute_failed_maps_compute_failed() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_COMPUTE_FAILED, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(DecodeError::ComputeFailed)); - } + use super::batch_ext_init_status_to_result; + use crate::error::llama_context_load_error::LlamaContextLoadError; #[test] - fn decode_allocation_failed_maps_not_enough_memory() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(DecodeError::NotEnoughMemory)); - } - - #[test] - fn decode_cxx_exception_without_a_message_is_a_contract_error() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - ); - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_decode", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); + fn initializing_a_batch_reports_each_null_pointer_as_a_contract_violation() { + for (status, detail) in [ + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_OK, + "success status contained a null batch", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_CTX_ARG, + "was given a null ctx argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_BATCH_ARG, + "was given a null out_batch argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_NULL_OUT_ERROR_ARG, + "was given a null out_error argument", + ), + ] { + assert_eq!( + batch_ext_init_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err(), + Some(LlamaContextLoadError::FfiContract(FfiContractError { + operation: "llama_rs_batch_ext_init", + detail, + })) + ); + } } #[test] - fn decode_nonzero_status_with_zero_code_is_contract_error() { - let result = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 0, - ptr::null_mut(), - ); - + fn initializing_a_batch_reports_memory_exhaustion() { assert_eq!( - result, - Err(DecodeError::FfiContract(crate::FfiContractError { - operation: "llama_rs_decode", - detail: "nonzero llama.cpp return status contained zero", - })) + [ + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_OUT_OF_MEMORY, + ] + .map(|status| batch_ext_init_status_to_result( + status, + ptr::null_mut(), + ptr::null_mut() + ) + .err()), + [ + Some(LlamaContextLoadError::NotEnoughMemory), + Some(LlamaContextLoadError::LlamaCppOutOfMemory), + ] ); } #[test] - fn decode_unknown_status_is_preserved() { - let result = decode_status_to_result(255, 0, ptr::null_mut()); - - assert_eq!( - result, - Err(DecodeError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_decode", - code: 255, - })) - ); - } + fn initializing_a_batch_reports_an_exception_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"batch allocation threw".as_ptr()) + }; - #[test] - fn decode_status_to_result_maps_null_ctx_arg() { - let outcome = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_CTX_ARG, - 0, - ptr::null_mut(), - ); assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_decode", - detail: "was given a null ctx argument", - } - .into() + batch_ext_init_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, ) + .err(), + Some(LlamaContextLoadError::Reported { + message: "batch allocation threw".to_owned() + }) ); } #[test] - fn decode_status_to_result_maps_null_out_error_arg() { - let outcome = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_NULL_OUT_ERROR_ARG, - 0, - ptr::null_mut(), - ); + fn initializing_a_batch_reports_an_exception_without_a_message_as_a_contract_violation() { assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_decode", - detail: "was given a null out_error argument", - } - .into() + batch_ext_init_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ptr::null_mut(), ) + .err(), + Some(LlamaContextLoadError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_batch_ext_init", + detail: "reported a thrown C++ exception without an error message", + } + ))) ); } #[test] - fn decode_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = decode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DECODE_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), + fn initializing_a_batch_preserves_an_unknown_status() { + assert_eq!( + batch_ext_init_status_to_result(255, ptr::null_mut(), ptr::null_mut()).err(), + Some(LlamaContextLoadError::FfiStatus(FfiStatusError { + operation: "llama_rs_batch_ext_init", + code: 255, + })) ); - assert_eq!(outcome.err(), Some(DecodeError::LlamaCppOutOfMemory)); } } #[cfg(test)] -mod encode_status_to_result_tests { +mod process_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; - use super::encode_status_to_result; - use crate::EncodeError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; - #[test] - fn encode_model_has_no_encoder_maps_model_has_no_encoder() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_MODEL_HAS_NO_ENCODER, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(EncodeError::ModelHasNoEncoder)); - } - - #[test] - fn encode_nonzero_code_maps_from_code() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 1, - ptr::null_mut(), - ); + use super::process_status_to_result; + use crate::error::decode_error::DecodeError; + use crate::error::encode_error::EncodeError; + use crate::error::process_failure::ProcessFailure; - assert_eq!(result, Err(EncodeError::NoKvCacheSlot)); - } - - #[test] - fn encode_out_of_memory_maps_encode_out_of_memory() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_OUT_OF_MEMORY, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(EncodeError::EncodeOutOfMemory)); - } - - #[test] - fn encode_compute_failed_maps_compute_failed() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_COMPUTE_FAILED, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(EncodeError::ComputeFailed)); - } - - #[test] - fn encode_allocation_failed_maps_not_enough_memory() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ); + const OPERATION: &str = "llama_rs_process"; - assert_eq!(result, Err(EncodeError::NotEnoughMemory)); + fn contract_violation(detail: &'static str) -> DecodeError { + ProcessFailure::from(FfiContractError { + operation: OPERATION, + detail, + }) + .into() } - #[test] - fn encode_cxx_exception_without_a_message_is_a_contract_error() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - ); - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); + fn missing_message(detail: &'static str) -> DecodeError { + ProcessFailure::from(CppStringError::Missing(FfiContractError { + operation: OPERATION, + detail, + })) + .into() } #[test] - fn encode_nonzero_status_with_zero_code_is_contract_error() { - let result = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 0, - ptr::null_mut(), - ); + fn every_status_without_a_return_code_or_message_maps_to_its_result() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_OK, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_CTX_ARG, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_BATCH_ARG, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_NULL_OUT_ERROR_ARG, + 255, + ]; assert_eq!( - result, - Err(EncodeError::FfiContract(crate::FfiContractError { - operation: "llama_rs_encode", - detail: "nonzero llama.cpp return status contained zero", - })) + statuses.map(|status| process_status_to_result::( + status, + 0, + ptr::null_mut() + )), + [ + Ok(()), + Err(contract_violation( + "nonzero llama.cpp return status contained zero" + )), + Err(ProcessFailure::NotEnoughMemory.into()), + Err(ProcessFailure::LlamaCppOutOfMemory.into()), + Err(missing_message( + "reported a thrown C++ exception without an error message" + )), + Err(contract_violation("was given a null ctx argument")), + Err(contract_violation("was given a null batch argument")), + Err(contract_violation( + "was given a null out_llama_cpp_return_code argument" + )), + Err(contract_violation("was given a null out_error argument")), + Err(ProcessFailure::from(FfiStatusError { + operation: OPERATION, + code: 255, + }) + .into()), + ] ); } #[test] - fn encode_unknown_status_is_preserved() { - let result = encode_status_to_result(255, 0, ptr::null_mut()); - + fn a_nonzero_llama_cpp_code_is_read_as_the_error_of_the_process_type() { assert_eq!( - result, - Err(EncodeError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_encode", - code: 255, - })) - ); - } - - #[test] - fn encode_status_to_result_maps_null_ctx_arg() { - let outcome = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_CTX_ARG, - 0, - ptr::null_mut(), + [ + process_status_to_result::( + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + 2, + ptr::null_mut() + ), + process_status_to_result::( + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + -3, + ptr::null_mut() + ), + ], + [Err(DecodeError::Aborted), Err(DecodeError::ComputeFailed)] ); assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null ctx argument", - } - .into() - ) - ); - } - - #[test] - fn encode_status_to_result_maps_null_model() { - let outcome = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_NULL_MODEL, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_encode", - detail: "was given a null model argument", - } - .into() - ) - ); - } - - #[test] - fn encode_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = encode_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_ENCODE_LLAMA_CPP_OUT_OF_MEMORY, - 0, - ptr::null_mut(), + process_status_to_result::( + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_RETURNED_NONZERO_CODE, + -2, + ptr::null_mut() + ), + Err(EncodeError::EncodeOutOfMemory) ); - assert_eq!(outcome.err(), Some(EncodeError::LlamaCppOutOfMemory)); } -} - -#[cfg(test)] -mod token_index_within_context_tests { - use super::token_index_within_context; - use crate::LogitsError; #[test] - fn token_index_beyond_context_size_maps_exceeds_context() { - let result = token_index_within_context(5, 4); + fn a_thrown_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"process threw".as_ptr()) }; assert_eq!( - result, - Err(LogitsError::TokenIndexExceedsContext { - token_index: 5, - context_size: 4, - }) + process_status_to_result::( + llama_cpp_bindings_sys::LLAMA_RS_PROCESS_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ), + Err(ProcessFailure::Reported { + message: "process threw".to_owned() + } + .into()) ); } - - #[test] - fn token_index_within_context_size_is_ok() { - assert!(token_index_within_context(2, 4).is_ok()); - } - - #[test] - fn token_index_negative_skips_context_check() { - assert!(token_index_within_context(-1, 4).is_ok()); - } } #[cfg(test)] -mod logits_slice_from_raw_parts_tests { - use std::ptr; +mod logits_slice_tests { + use std::ptr::NonNull; - use super::logits_slice_from_raw_parts; - use crate::LogitsError; - - #[test] - fn logits_slice_from_null_data_maps_null_logits() { - let result = unsafe { logits_slice_from_raw_parts(ptr::null(), 4) }; - - assert_eq!(result, Err(LogitsError::NullLogits)); - } + use super::logits_slice; + use crate::error::logits_error::LogitsError; #[test] fn logits_slice_from_negative_vocab_maps_vocab_size_overflow() { let logit_value = 0.0_f32; - let result = unsafe { logits_slice_from_raw_parts(&raw const logit_value, -1) }; + let result = unsafe { logits_slice(NonNull::from(&logit_value), -1) }; let conversion_error = usize::try_from(-1_i32).unwrap_err(); diff --git a/llama-cpp-bindings/src/context/context_limits.rs b/llama-cpp-bindings/src/context/context_limits.rs new file mode 100644 index 000000000..95ec33cff --- /dev/null +++ b/llama-cpp-bindings/src/context/context_limits.rs @@ -0,0 +1,147 @@ +use std::num::NonZeroU32; + +use llama_cpp_bindings_sys::llama_seq_id; + +use crate::context::LlamaContext; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; +use crate::token::LlamaToken; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ContextLimits { + pub n_batch: u32, + pub n_ubatch: u32, + pub n_seq_max: u32, + pub n_vocab: i32, +} + +impl ContextLimits { + #[must_use] + pub const fn of(context: &LlamaContext) -> Self { + context.limits + } + + /// # Errors + /// + /// Returns [`TokenOutsideVocabulary`] when `token` is not in the vocabulary of the model. + pub fn ensure_token_in_vocabulary( + &self, + token: LlamaToken, + ) -> Result { + TokenOutsideVocabulary::check(token, self.n_vocab) + } + + /// # Errors + /// + /// Returns [`SequenceIdOutOfRange`] when `seq_id` names no sequence of the context. + pub fn ensure_sequence_id(&self, seq_id: llama_seq_id) -> Result<(), SequenceIdOutOfRange> { + if u32::try_from(seq_id).is_ok_and(|sequence| sequence < self.n_seq_max) { + Ok(()) + } else { + Err(SequenceIdOutOfRange { + seq_id, + n_seq_max: self.n_seq_max, + }) + } + } + + /// Returns how many tokens a single decode of `batch_tokens` evaluates: the smaller of + /// `batch_tokens` and the context's micro batch. + #[must_use] + pub const fn micro_batch_tokens(&self, batch_tokens: NonZeroU32) -> u32 { + let batch_tokens = batch_tokens.get(); + + if batch_tokens < self.n_ubatch { + batch_tokens + } else { + self.n_ubatch + } + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use super::ContextLimits; + use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; + use crate::token::LlamaToken; + + const LIMITS: ContextLimits = ContextLimits { + n_batch: 512, + n_ubatch: 128, + n_seq_max: 2, + n_vocab: 100, + }; + + #[test] + fn the_last_sequence_of_the_context_is_accepted() { + assert_eq!(LIMITS.ensure_sequence_id(1), Ok(())); + } + + #[test] + fn a_negative_sequence_id_is_rejected() { + assert_eq!( + LIMITS.ensure_sequence_id(-1), + Err(SequenceIdOutOfRange { + seq_id: -1, + n_seq_max: 2, + }) + ); + } + + #[test] + fn a_sequence_id_past_the_last_sequence_is_rejected() { + assert_eq!( + LIMITS.ensure_sequence_id(2), + Err(SequenceIdOutOfRange { + seq_id: 2, + n_seq_max: 2, + }) + ); + } + + #[test] + fn the_last_token_of_the_vocabulary_is_accepted() { + assert_eq!( + LIMITS.ensure_token_in_vocabulary(LlamaToken::new(99)), + Ok(LlamaToken::new(99)) + ); + } + + #[test] + fn a_token_past_the_vocabulary_is_rejected() { + assert_eq!( + LIMITS.ensure_token_in_vocabulary(LlamaToken::new(100)), + Err(TokenOutsideVocabulary { + token: LlamaToken::new(100), + n_vocab: 100, + }) + ); + } + + #[test] + fn a_negative_token_is_rejected() { + assert_eq!( + LIMITS.ensure_token_in_vocabulary(LlamaToken::new(-1)), + Err(TokenOutsideVocabulary { + token: LlamaToken::new(-1), + n_vocab: 100, + }) + ); + } + + #[test] + fn a_batch_smaller_than_the_micro_batch_is_decoded_whole() { + assert_eq!(LIMITS.micro_batch_tokens(NonZeroU32::new(64).unwrap()), 64); + } + + #[test] + fn a_batch_larger_than_the_micro_batch_is_decoded_in_micro_batches() { + assert_eq!( + LIMITS.micro_batch_tokens(NonZeroU32::new(256).unwrap()), + 128 + ); + } +} diff --git a/llama-cpp-bindings/src/context/decoded_output_rows.rs b/llama-cpp-bindings/src/context/decoded_output_rows.rs new file mode 100644 index 000000000..ffe792668 --- /dev/null +++ b/llama-cpp-bindings/src/context/decoded_output_rows.rs @@ -0,0 +1,39 @@ +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum DecodedOutputRows { + BatchIndices(Vec), + LastRow, +} + +impl DecodedOutputRows { + /// The rows a llama.cpp helper writes when it decodes on its own batches, which output at most + /// the last row and only when asked to. + #[must_use] + pub const fn of_helper_decode(logits_last: bool) -> Self { + if logits_last { + Self::LastRow + } else { + Self::BatchIndices(Vec::new()) + } + } +} + +#[cfg(test)] +mod tests { + use super::DecodedOutputRows; + + #[test] + fn a_helper_decode_with_logits_last_outputs_its_last_row() { + assert_eq!( + DecodedOutputRows::of_helper_decode(true), + DecodedOutputRows::LastRow + ); + } + + #[test] + fn a_helper_decode_without_logits_last_outputs_no_row() { + assert_eq!( + DecodedOutputRows::of_helper_decode(false), + DecodedOutputRows::BatchIndices(Vec::new()) + ); + } +} diff --git a/llama-cpp-bindings/src/context/decoded_outputs.rs b/llama-cpp-bindings/src/context/decoded_outputs.rs new file mode 100644 index 000000000..f701e3321 --- /dev/null +++ b/llama-cpp-bindings/src/context/decoded_outputs.rs @@ -0,0 +1,135 @@ +use crate::context::decoded_output_rows::DecodedOutputRows; +use crate::context::llama_pooling_type::LlamaPoolingType; +use crate::error::logits_error::LogitsError; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DecodedOutputs { + pub rows: DecodedOutputRows, + pub pooling_type: LlamaPoolingType, +} + +impl DecodedOutputs { + /// Accepts the batch index of an output token, or a negative index counting back from the + /// last output, the way `llama_get_logits_ith` resolves it. + /// + /// # Errors + /// + /// Returns [`LogitsError::TokenNotInitialized`] when the index names no output of the last + /// decode. + pub fn ensure_output(&self, token_index: i32) -> Result<(), LogitsError> { + let names_an_output = match &self.rows { + DecodedOutputRows::BatchIndices(output_token_indices) if token_index >= 0 => { + output_token_indices.binary_search(&token_index).is_ok() + } + DecodedOutputRows::BatchIndices(output_token_indices) => { + token_index.unsigned_abs() as usize <= output_token_indices.len() + } + DecodedOutputRows::LastRow => token_index == -1, + }; + + if names_an_output { + Ok(()) + } else { + Err(LogitsError::TokenNotInitialized(token_index)) + } + } + + #[must_use] + pub const fn missing_logits_error(&self) -> LogitsError { + match self.pooling_type { + LlamaPoolingType::None | LlamaPoolingType::Unspecified => { + LogitsError::GraphProducesNoLogits + } + pooling_type @ (LlamaPoolingType::Mean + | LlamaPoolingType::Cls + | LlamaPoolingType::Last + | LlamaPoolingType::Rank) => { + LogitsError::PooledEmbeddingsProduceNoLogits { pooling_type } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::DecodedOutputs; + use crate::context::decoded_output_rows::DecodedOutputRows; + use crate::context::llama_pooling_type::LlamaPoolingType; + use crate::error::logits_error::LogitsError; + + fn outputs_at(output_token_indices: Vec) -> DecodedOutputs { + DecodedOutputs { + rows: DecodedOutputRows::BatchIndices(output_token_indices), + pooling_type: LlamaPoolingType::None, + } + } + + fn last_row_only() -> DecodedOutputs { + DecodedOutputs { + rows: DecodedOutputRows::LastRow, + pooling_type: LlamaPoolingType::None, + } + } + + #[test] + fn a_batch_index_marked_as_output_names_an_output() { + assert_eq!(outputs_at(vec![2, 5]).ensure_output(5), Ok(())); + } + + #[test] + fn a_batch_index_not_marked_as_output_is_refused() { + assert_eq!( + outputs_at(vec![2, 5]).ensure_output(3), + Err(LogitsError::TokenNotInitialized(3)) + ); + } + + #[test] + fn a_negative_index_counts_back_from_the_last_output() { + assert_eq!(outputs_at(vec![2, 5]).ensure_output(-2), Ok(())); + } + + #[test] + fn a_negative_index_beyond_the_first_output_is_refused() { + assert_eq!( + outputs_at(vec![2, 5]).ensure_output(-3), + Err(LogitsError::TokenNotInitialized(-3)) + ); + } + + #[test] + fn a_decode_outputting_only_its_last_row_names_it_from_the_end() { + assert_eq!(last_row_only().ensure_output(-1), Ok(())); + } + + #[test] + fn a_decode_outputting_only_its_last_row_refuses_batch_indices() { + assert_eq!( + last_row_only().ensure_output(0), + Err(LogitsError::TokenNotInitialized(0)) + ); + } + + #[test] + fn missing_logits_of_a_pooled_decode_are_attributed_to_pooling() { + let decoded = DecodedOutputs { + rows: DecodedOutputRows::BatchIndices(vec![0]), + pooling_type: LlamaPoolingType::Mean, + }; + + assert_eq!( + decoded.missing_logits_error(), + LogitsError::PooledEmbeddingsProduceNoLogits { + pooling_type: LlamaPoolingType::Mean + } + ); + } + + #[test] + fn missing_logits_of_an_unpooled_decode_are_attributed_to_the_graph() { + assert_eq!( + outputs_at(vec![0]).missing_logits_error(), + LogitsError::GraphProducesNoLogits + ); + } +} diff --git a/llama-cpp-bindings/src/context/kv_cache_type.rs b/llama-cpp-bindings/src/context/kv_cache_type.rs index 150b5625c..4f8d4389c 100644 --- a/llama-cpp-bindings/src/context/kv_cache_type.rs +++ b/llama-cpp-bindings/src/context/kv_cache_type.rs @@ -1,51 +1,41 @@ -#[expect( - non_camel_case_types, - reason = "variant names mirror llama.cpp's `enum ggml_type` symbol names verbatim so they can \ - be matched 1:1 against the C ABI without a translation table" -)] -#[expect( - missing_docs, - reason = "each variant denotes a quantisation flavour whose semantics are defined upstream in \ - ggml; restating the upstream spec inline would risk drifting from the source of truth" -)] #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum KvCacheType { Unknown(llama_cpp_bindings_sys::ggml_type), F32, F16, - Q4_0, - Q4_1, - Q5_0, - Q5_1, - Q8_0, - Q8_1, - Q2_K, - Q3_K, - Q4_K, - Q5_K, - Q6_K, - Q8_K, - IQ2_XXS, - IQ2_XS, - IQ3_XXS, - IQ1_S, - IQ4_NL, - IQ3_S, - IQ2_S, - IQ4_XS, + Q40, + Q41, + Q50, + Q51, + Q80, + Q81, + Q2K, + Q3K, + Q4K, + Q5K, + Q6K, + Q8K, + Iq2Xxs, + Iq2Xs, + Iq3Xxs, + Iq1S, + Iq4Nl, + Iq3S, + Iq2S, + Iq4Xs, I8, I16, I32, I64, F64, - IQ1_M, - BF16, - TQ1_0, - TQ2_0, - MXFP4, - NVFP4, - Q1_0, - Q2_0, + Iq1M, + Bf16, + Tq10, + Tq20, + Mxfp4, + Nvfp4, + Q10, + Q20, } impl From for llama_cpp_bindings_sys::ggml_type { @@ -54,39 +44,39 @@ impl From for llama_cpp_bindings_sys::ggml_type { KvCacheType::Unknown(raw) => raw, KvCacheType::F32 => llama_cpp_bindings_sys::GGML_TYPE_F32, KvCacheType::F16 => llama_cpp_bindings_sys::GGML_TYPE_F16, - KvCacheType::Q4_0 => llama_cpp_bindings_sys::GGML_TYPE_Q4_0, - KvCacheType::Q4_1 => llama_cpp_bindings_sys::GGML_TYPE_Q4_1, - KvCacheType::Q5_0 => llama_cpp_bindings_sys::GGML_TYPE_Q5_0, - KvCacheType::Q5_1 => llama_cpp_bindings_sys::GGML_TYPE_Q5_1, - KvCacheType::Q8_0 => llama_cpp_bindings_sys::GGML_TYPE_Q8_0, - KvCacheType::Q8_1 => llama_cpp_bindings_sys::GGML_TYPE_Q8_1, - KvCacheType::Q2_K => llama_cpp_bindings_sys::GGML_TYPE_Q2_K, - KvCacheType::Q3_K => llama_cpp_bindings_sys::GGML_TYPE_Q3_K, - KvCacheType::Q4_K => llama_cpp_bindings_sys::GGML_TYPE_Q4_K, - KvCacheType::Q5_K => llama_cpp_bindings_sys::GGML_TYPE_Q5_K, - KvCacheType::Q6_K => llama_cpp_bindings_sys::GGML_TYPE_Q6_K, - KvCacheType::Q8_K => llama_cpp_bindings_sys::GGML_TYPE_Q8_K, - KvCacheType::IQ2_XXS => llama_cpp_bindings_sys::GGML_TYPE_IQ2_XXS, - KvCacheType::IQ2_XS => llama_cpp_bindings_sys::GGML_TYPE_IQ2_XS, - KvCacheType::IQ3_XXS => llama_cpp_bindings_sys::GGML_TYPE_IQ3_XXS, - KvCacheType::IQ1_S => llama_cpp_bindings_sys::GGML_TYPE_IQ1_S, - KvCacheType::IQ4_NL => llama_cpp_bindings_sys::GGML_TYPE_IQ4_NL, - KvCacheType::IQ3_S => llama_cpp_bindings_sys::GGML_TYPE_IQ3_S, - KvCacheType::IQ2_S => llama_cpp_bindings_sys::GGML_TYPE_IQ2_S, - KvCacheType::IQ4_XS => llama_cpp_bindings_sys::GGML_TYPE_IQ4_XS, + KvCacheType::Q40 => llama_cpp_bindings_sys::GGML_TYPE_Q4_0, + KvCacheType::Q41 => llama_cpp_bindings_sys::GGML_TYPE_Q4_1, + KvCacheType::Q50 => llama_cpp_bindings_sys::GGML_TYPE_Q5_0, + KvCacheType::Q51 => llama_cpp_bindings_sys::GGML_TYPE_Q5_1, + KvCacheType::Q80 => llama_cpp_bindings_sys::GGML_TYPE_Q8_0, + KvCacheType::Q81 => llama_cpp_bindings_sys::GGML_TYPE_Q8_1, + KvCacheType::Q2K => llama_cpp_bindings_sys::GGML_TYPE_Q2_K, + KvCacheType::Q3K => llama_cpp_bindings_sys::GGML_TYPE_Q3_K, + KvCacheType::Q4K => llama_cpp_bindings_sys::GGML_TYPE_Q4_K, + KvCacheType::Q5K => llama_cpp_bindings_sys::GGML_TYPE_Q5_K, + KvCacheType::Q6K => llama_cpp_bindings_sys::GGML_TYPE_Q6_K, + KvCacheType::Q8K => llama_cpp_bindings_sys::GGML_TYPE_Q8_K, + KvCacheType::Iq2Xxs => llama_cpp_bindings_sys::GGML_TYPE_IQ2_XXS, + KvCacheType::Iq2Xs => llama_cpp_bindings_sys::GGML_TYPE_IQ2_XS, + KvCacheType::Iq3Xxs => llama_cpp_bindings_sys::GGML_TYPE_IQ3_XXS, + KvCacheType::Iq1S => llama_cpp_bindings_sys::GGML_TYPE_IQ1_S, + KvCacheType::Iq4Nl => llama_cpp_bindings_sys::GGML_TYPE_IQ4_NL, + KvCacheType::Iq3S => llama_cpp_bindings_sys::GGML_TYPE_IQ3_S, + KvCacheType::Iq2S => llama_cpp_bindings_sys::GGML_TYPE_IQ2_S, + KvCacheType::Iq4Xs => llama_cpp_bindings_sys::GGML_TYPE_IQ4_XS, KvCacheType::I8 => llama_cpp_bindings_sys::GGML_TYPE_I8, KvCacheType::I16 => llama_cpp_bindings_sys::GGML_TYPE_I16, KvCacheType::I32 => llama_cpp_bindings_sys::GGML_TYPE_I32, KvCacheType::I64 => llama_cpp_bindings_sys::GGML_TYPE_I64, KvCacheType::F64 => llama_cpp_bindings_sys::GGML_TYPE_F64, - KvCacheType::IQ1_M => llama_cpp_bindings_sys::GGML_TYPE_IQ1_M, - KvCacheType::BF16 => llama_cpp_bindings_sys::GGML_TYPE_BF16, - KvCacheType::TQ1_0 => llama_cpp_bindings_sys::GGML_TYPE_TQ1_0, - KvCacheType::TQ2_0 => llama_cpp_bindings_sys::GGML_TYPE_TQ2_0, - KvCacheType::MXFP4 => llama_cpp_bindings_sys::GGML_TYPE_MXFP4, - KvCacheType::NVFP4 => llama_cpp_bindings_sys::GGML_TYPE_NVFP4, - KvCacheType::Q1_0 => llama_cpp_bindings_sys::GGML_TYPE_Q1_0, - KvCacheType::Q2_0 => llama_cpp_bindings_sys::GGML_TYPE_Q2_0, + KvCacheType::Iq1M => llama_cpp_bindings_sys::GGML_TYPE_IQ1_M, + KvCacheType::Bf16 => llama_cpp_bindings_sys::GGML_TYPE_BF16, + KvCacheType::Tq10 => llama_cpp_bindings_sys::GGML_TYPE_TQ1_0, + KvCacheType::Tq20 => llama_cpp_bindings_sys::GGML_TYPE_TQ2_0, + KvCacheType::Mxfp4 => llama_cpp_bindings_sys::GGML_TYPE_MXFP4, + KvCacheType::Nvfp4 => llama_cpp_bindings_sys::GGML_TYPE_NVFP4, + KvCacheType::Q10 => llama_cpp_bindings_sys::GGML_TYPE_Q1_0, + KvCacheType::Q20 => llama_cpp_bindings_sys::GGML_TYPE_Q2_0, } } } @@ -94,41 +84,41 @@ impl From for llama_cpp_bindings_sys::ggml_type { impl From for KvCacheType { fn from(value: llama_cpp_bindings_sys::ggml_type) -> Self { match value { - x if x == llama_cpp_bindings_sys::GGML_TYPE_F32 => Self::F32, - x if x == llama_cpp_bindings_sys::GGML_TYPE_F16 => Self::F16, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q4_0 => Self::Q4_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q4_1 => Self::Q4_1, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q5_0 => Self::Q5_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q5_1 => Self::Q5_1, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q8_0 => Self::Q8_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q8_1 => Self::Q8_1, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q2_K => Self::Q2_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q3_K => Self::Q3_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q4_K => Self::Q4_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q5_K => Self::Q5_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q6_K => Self::Q6_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q8_K => Self::Q8_K, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ2_XXS => Self::IQ2_XXS, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ2_XS => Self::IQ2_XS, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ3_XXS => Self::IQ3_XXS, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ1_S => Self::IQ1_S, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ4_NL => Self::IQ4_NL, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ3_S => Self::IQ3_S, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ2_S => Self::IQ2_S, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ4_XS => Self::IQ4_XS, - x if x == llama_cpp_bindings_sys::GGML_TYPE_I8 => Self::I8, - x if x == llama_cpp_bindings_sys::GGML_TYPE_I16 => Self::I16, - x if x == llama_cpp_bindings_sys::GGML_TYPE_I32 => Self::I32, - x if x == llama_cpp_bindings_sys::GGML_TYPE_I64 => Self::I64, - x if x == llama_cpp_bindings_sys::GGML_TYPE_F64 => Self::F64, - x if x == llama_cpp_bindings_sys::GGML_TYPE_IQ1_M => Self::IQ1_M, - x if x == llama_cpp_bindings_sys::GGML_TYPE_BF16 => Self::BF16, - x if x == llama_cpp_bindings_sys::GGML_TYPE_TQ1_0 => Self::TQ1_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_TQ2_0 => Self::TQ2_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_MXFP4 => Self::MXFP4, - x if x == llama_cpp_bindings_sys::GGML_TYPE_NVFP4 => Self::NVFP4, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q1_0 => Self::Q1_0, - x if x == llama_cpp_bindings_sys::GGML_TYPE_Q2_0 => Self::Q2_0, + llama_cpp_bindings_sys::GGML_TYPE_F32 => Self::F32, + llama_cpp_bindings_sys::GGML_TYPE_F16 => Self::F16, + llama_cpp_bindings_sys::GGML_TYPE_Q4_0 => Self::Q40, + llama_cpp_bindings_sys::GGML_TYPE_Q4_1 => Self::Q41, + llama_cpp_bindings_sys::GGML_TYPE_Q5_0 => Self::Q50, + llama_cpp_bindings_sys::GGML_TYPE_Q5_1 => Self::Q51, + llama_cpp_bindings_sys::GGML_TYPE_Q8_0 => Self::Q80, + llama_cpp_bindings_sys::GGML_TYPE_Q8_1 => Self::Q81, + llama_cpp_bindings_sys::GGML_TYPE_Q2_K => Self::Q2K, + llama_cpp_bindings_sys::GGML_TYPE_Q3_K => Self::Q3K, + llama_cpp_bindings_sys::GGML_TYPE_Q4_K => Self::Q4K, + llama_cpp_bindings_sys::GGML_TYPE_Q5_K => Self::Q5K, + llama_cpp_bindings_sys::GGML_TYPE_Q6_K => Self::Q6K, + llama_cpp_bindings_sys::GGML_TYPE_Q8_K => Self::Q8K, + llama_cpp_bindings_sys::GGML_TYPE_IQ2_XXS => Self::Iq2Xxs, + llama_cpp_bindings_sys::GGML_TYPE_IQ2_XS => Self::Iq2Xs, + llama_cpp_bindings_sys::GGML_TYPE_IQ3_XXS => Self::Iq3Xxs, + llama_cpp_bindings_sys::GGML_TYPE_IQ1_S => Self::Iq1S, + llama_cpp_bindings_sys::GGML_TYPE_IQ4_NL => Self::Iq4Nl, + llama_cpp_bindings_sys::GGML_TYPE_IQ3_S => Self::Iq3S, + llama_cpp_bindings_sys::GGML_TYPE_IQ2_S => Self::Iq2S, + llama_cpp_bindings_sys::GGML_TYPE_IQ4_XS => Self::Iq4Xs, + llama_cpp_bindings_sys::GGML_TYPE_I8 => Self::I8, + llama_cpp_bindings_sys::GGML_TYPE_I16 => Self::I16, + llama_cpp_bindings_sys::GGML_TYPE_I32 => Self::I32, + llama_cpp_bindings_sys::GGML_TYPE_I64 => Self::I64, + llama_cpp_bindings_sys::GGML_TYPE_F64 => Self::F64, + llama_cpp_bindings_sys::GGML_TYPE_IQ1_M => Self::Iq1M, + llama_cpp_bindings_sys::GGML_TYPE_BF16 => Self::Bf16, + llama_cpp_bindings_sys::GGML_TYPE_TQ1_0 => Self::Tq10, + llama_cpp_bindings_sys::GGML_TYPE_TQ2_0 => Self::Tq20, + llama_cpp_bindings_sys::GGML_TYPE_MXFP4 => Self::Mxfp4, + llama_cpp_bindings_sys::GGML_TYPE_NVFP4 => Self::Nvfp4, + llama_cpp_bindings_sys::GGML_TYPE_Q1_0 => Self::Q10, + llama_cpp_bindings_sys::GGML_TYPE_Q2_0 => Self::Q20, _ => Self::Unknown(value), } } @@ -153,7 +143,8 @@ mod tests { let upstream_name = unsafe { CStr::from_ptr(llama_cpp_bindings_sys::ggml_type_name(raw)) } - .to_string_lossy(); + .to_str() + .expect("ggml type names are ASCII"); assert_ne!( KvCacheType::from(raw), @@ -177,50 +168,60 @@ mod tests { } #[test] - fn kv_cache_type_all_known_variants_roundtrip() { + fn every_kv_cache_type_converts_to_the_matching_ggml_type_and_back() { let all_variants = [ KvCacheType::F32, KvCacheType::F16, - KvCacheType::Q4_0, - KvCacheType::Q4_1, - KvCacheType::Q5_0, - KvCacheType::Q5_1, - KvCacheType::Q8_0, - KvCacheType::Q8_1, - KvCacheType::Q2_K, - KvCacheType::Q3_K, - KvCacheType::Q4_K, - KvCacheType::Q5_K, - KvCacheType::Q6_K, - KvCacheType::Q8_K, - KvCacheType::IQ2_XXS, - KvCacheType::IQ2_XS, - KvCacheType::IQ3_XXS, - KvCacheType::IQ1_S, - KvCacheType::IQ4_NL, - KvCacheType::IQ3_S, - KvCacheType::IQ2_S, - KvCacheType::IQ4_XS, + KvCacheType::Q40, + KvCacheType::Q41, + KvCacheType::Q50, + KvCacheType::Q51, + KvCacheType::Q80, + KvCacheType::Q81, + KvCacheType::Q2K, + KvCacheType::Q3K, + KvCacheType::Q4K, + KvCacheType::Q5K, + KvCacheType::Q6K, + KvCacheType::Q8K, + KvCacheType::Iq2Xxs, + KvCacheType::Iq2Xs, + KvCacheType::Iq3Xxs, + KvCacheType::Iq1S, + KvCacheType::Iq4Nl, + KvCacheType::Iq3S, + KvCacheType::Iq2S, + KvCacheType::Iq4Xs, KvCacheType::I8, KvCacheType::I16, KvCacheType::I32, KvCacheType::I64, KvCacheType::F64, - KvCacheType::IQ1_M, - KvCacheType::BF16, - KvCacheType::TQ1_0, - KvCacheType::TQ2_0, - KvCacheType::MXFP4, - KvCacheType::NVFP4, - KvCacheType::Q1_0, - KvCacheType::Q2_0, + KvCacheType::Iq1M, + KvCacheType::Bf16, + KvCacheType::Tq10, + KvCacheType::Tq20, + KvCacheType::Mxfp4, + KvCacheType::Nvfp4, + KvCacheType::Q10, + KvCacheType::Q20, ]; - for variant in all_variants { - let ggml_type: llama_cpp_bindings_sys::ggml_type = variant.into(); - let back = KvCacheType::from(ggml_type); + let ggml_types = all_variants.map(llama_cpp_bindings_sys::ggml_type::from); - assert_eq!(back, variant); - } + assert_eq!( + ggml_types.map(|ggml_type| { + unsafe { CStr::from_ptr(llama_cpp_bindings_sys::ggml_type_name(ggml_type)) } + .to_str() + .unwrap() + }), + [ + "f32", "f16", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "q8_1", "q2_K", "q3_K", + "q4_K", "q5_K", "q6_K", "q8_K", "iq2_xxs", "iq2_xs", "iq3_xxs", "iq1_s", "iq4_nl", + "iq3_s", "iq2_s", "iq4_xs", "i8", "i16", "i32", "i64", "f64", "iq1_m", "bf16", + "tq1_0", "tq2_0", "mxfp4", "nvfp4", "q1_0", "q2_0", + ] + ); + assert_eq!(ggml_types.map(KvCacheType::from), all_variants); } } diff --git a/llama-cpp-bindings/src/context/llama_attention_type.rs b/llama-cpp-bindings/src/context/llama_attention_type.rs index 79b9f66f6..5a233573f 100644 --- a/llama-cpp-bindings/src/context/llama_attention_type.rs +++ b/llama-cpp-bindings/src/context/llama_attention_type.rs @@ -1,27 +1,35 @@ -#[repr(i8)] +use crate::error::unknown_attention_type::UnknownAttentionType; + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum LlamaAttentionType { - Unspecified = -1, - Causal = 0, - NonCausal = 1, + Unspecified, + Causal, + NonCausal, } -impl From for LlamaAttentionType { - fn from(value: i32) -> Self { +impl TryFrom for LlamaAttentionType { + type Error = UnknownAttentionType; + + fn try_from(value: llama_cpp_bindings_sys::llama_attention_type) -> Result { match value { - 0 => Self::Causal, - 1 => Self::NonCausal, - _ => Self::Unspecified, + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_UNSPECIFIED => Ok(Self::Unspecified), + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_CAUSAL => Ok(Self::Causal), + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_NON_CAUSAL => Ok(Self::NonCausal), + unknown => Err(UnknownAttentionType(unknown)), } } } -impl From for i32 { +impl From for llama_cpp_bindings_sys::llama_attention_type { fn from(value: LlamaAttentionType) -> Self { match value { - LlamaAttentionType::Causal => 0, - LlamaAttentionType::NonCausal => 1, - LlamaAttentionType::Unspecified => -1, + LlamaAttentionType::Unspecified => { + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_UNSPECIFIED + } + LlamaAttentionType::Causal => llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_CAUSAL, + LlamaAttentionType::NonCausal => { + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_NON_CAUSAL + } } } } @@ -29,31 +37,40 @@ impl From for i32 { #[cfg(test)] mod tests { use super::LlamaAttentionType; + use crate::error::unknown_attention_type::UnknownAttentionType; #[test] - fn attention_type_unknown_defaults_to_unspecified() { + fn every_llama_cpp_value_converts_to_its_variant_and_back() { + let raw_values = [ + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_UNSPECIFIED, + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_CAUSAL, + llama_cpp_bindings_sys::LLAMA_ATTENTION_TYPE_NON_CAUSAL, + ]; + assert_eq!( - LlamaAttentionType::from(99), - LlamaAttentionType::Unspecified + raw_values.map(LlamaAttentionType::try_from), + [ + Ok(LlamaAttentionType::Unspecified), + Ok(LlamaAttentionType::Causal), + Ok(LlamaAttentionType::NonCausal), + ] ); assert_eq!( - LlamaAttentionType::from(-50), - LlamaAttentionType::Unspecified + [ + LlamaAttentionType::Unspecified, + LlamaAttentionType::Causal, + LlamaAttentionType::NonCausal, + ] + .map(llama_cpp_bindings_sys::llama_attention_type::from), + raw_values ); } #[test] - fn attention_type_roundtrip_all_variants() { - for (raw, expected) in [ - (-1, LlamaAttentionType::Unspecified), - (0, LlamaAttentionType::Causal), - (1, LlamaAttentionType::NonCausal), - ] { - let from_raw = LlamaAttentionType::from(raw); - assert_eq!(from_raw, expected); - - let back_to_raw: i32 = from_raw.into(); - assert_eq!(back_to_raw, raw); - } + fn a_value_llama_cpp_does_not_define_is_refused() { + assert_eq!( + LlamaAttentionType::try_from(99), + Err(UnknownAttentionType(99)) + ); } } diff --git a/llama-cpp-bindings/src/context/llama_pooling_type.rs b/llama-cpp-bindings/src/context/llama_pooling_type.rs index 651216f39..ce1983f3c 100644 --- a/llama-cpp-bindings/src/context/llama_pooling_type.rs +++ b/llama-cpp-bindings/src/context/llama_pooling_type.rs @@ -1,36 +1,40 @@ -#[repr(i8)] +use crate::error::unknown_pooling_type::UnknownPoolingType; + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum LlamaPoolingType { - Unspecified = -1, - None = 0, - Mean = 1, - Cls = 2, - Last = 3, - Rank = 4, + Unspecified, + None, + Mean, + Cls, + Last, + Rank, } -impl From for LlamaPoolingType { - fn from(value: i32) -> Self { +impl TryFrom for LlamaPoolingType { + type Error = UnknownPoolingType; + + fn try_from(value: llama_cpp_bindings_sys::llama_pooling_type) -> Result { match value { - 0 => Self::None, - 1 => Self::Mean, - 2 => Self::Cls, - 3 => Self::Last, - 4 => Self::Rank, - _ => Self::Unspecified, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_UNSPECIFIED => Ok(Self::Unspecified), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_NONE => Ok(Self::None), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_MEAN => Ok(Self::Mean), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_CLS => Ok(Self::Cls), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_LAST => Ok(Self::Last), + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_RANK => Ok(Self::Rank), + unknown => Err(UnknownPoolingType(unknown)), } } } -impl From for i32 { +impl From for llama_cpp_bindings_sys::llama_pooling_type { fn from(value: LlamaPoolingType) -> Self { match value { - LlamaPoolingType::None => 0, - LlamaPoolingType::Mean => 1, - LlamaPoolingType::Cls => 2, - LlamaPoolingType::Last => 3, - LlamaPoolingType::Rank => 4, - LlamaPoolingType::Unspecified => -1, + LlamaPoolingType::Unspecified => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_UNSPECIFIED, + LlamaPoolingType::None => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_NONE, + LlamaPoolingType::Mean => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_MEAN, + LlamaPoolingType::Cls => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_CLS, + LlamaPoolingType::Last => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_LAST, + LlamaPoolingType::Rank => llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_RANK, } } } @@ -38,28 +42,46 @@ impl From for i32 { #[cfg(test)] mod tests { use super::LlamaPoolingType; + use crate::error::unknown_pooling_type::UnknownPoolingType; #[test] - fn pooling_type_unknown_defaults_to_unspecified() { - assert_eq!(LlamaPoolingType::from(99), LlamaPoolingType::Unspecified); - assert_eq!(LlamaPoolingType::from(-50), LlamaPoolingType::Unspecified); + fn every_llama_cpp_value_converts_to_its_variant_and_back() { + let raw_values = [ + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_UNSPECIFIED, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_NONE, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_MEAN, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_CLS, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_LAST, + llama_cpp_bindings_sys::LLAMA_POOLING_TYPE_RANK, + ]; + + assert_eq!( + raw_values.map(LlamaPoolingType::try_from), + [ + Ok(LlamaPoolingType::Unspecified), + Ok(LlamaPoolingType::None), + Ok(LlamaPoolingType::Mean), + Ok(LlamaPoolingType::Cls), + Ok(LlamaPoolingType::Last), + Ok(LlamaPoolingType::Rank), + ] + ); + assert_eq!( + [ + LlamaPoolingType::Unspecified, + LlamaPoolingType::None, + LlamaPoolingType::Mean, + LlamaPoolingType::Cls, + LlamaPoolingType::Last, + LlamaPoolingType::Rank, + ] + .map(llama_cpp_bindings_sys::llama_pooling_type::from), + raw_values + ); } #[test] - fn pooling_type_roundtrip_all_variants() { - for (raw, expected) in [ - (-1, LlamaPoolingType::Unspecified), - (0, LlamaPoolingType::None), - (1, LlamaPoolingType::Mean), - (2, LlamaPoolingType::Cls), - (3, LlamaPoolingType::Last), - (4, LlamaPoolingType::Rank), - ] { - let from_raw = LlamaPoolingType::from(raw); - assert_eq!(from_raw, expected); - - let back_to_raw: i32 = from_raw.into(); - assert_eq!(back_to_raw, raw); - } + fn a_value_llama_cpp_does_not_define_is_refused() { + assert_eq!(LlamaPoolingType::try_from(99), Err(UnknownPoolingType(99))); } } diff --git a/llama-cpp-bindings/src/context/llama_state_seq_flags.rs b/llama-cpp-bindings/src/context/llama_state_seq_flags.rs index cbe5ccc09..98c3a26f2 100644 --- a/llama-cpp-bindings/src/context/llama_state_seq_flags.rs +++ b/llama-cpp-bindings/src/context/llama_state_seq_flags.rs @@ -1,3 +1,5 @@ +use std::ops::BitOr; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct LlamaStateSeqFlags { flags: u32, @@ -22,7 +24,7 @@ impl LlamaStateSeqFlags { } } -impl std::ops::BitOr for LlamaStateSeqFlags { +impl BitOr for LlamaStateSeqFlags { type Output = Self; fn bitor(self, rhs: Self) -> Self { diff --git a/llama-cpp-bindings/src/context/load_seq_state_error.rs b/llama-cpp-bindings/src/context/load_seq_state_error.rs index b3e27983b..a25439e65 100644 --- a/llama-cpp-bindings/src/context/load_seq_state_error.rs +++ b/llama-cpp-bindings/src/context/load_seq_state_error.rs @@ -11,7 +11,4 @@ pub enum LoadSeqStateError { #[error("failed to convert path {0} to str")] PathToStrError(PathBuf), - - #[error("max_length is not large enough to hold {n_out} (was {max_tokens})")] - InsufficientMaxLength { n_out: usize, max_tokens: usize }, } diff --git a/llama-cpp-bindings/src/context/load_session_error.rs b/llama-cpp-bindings/src/context/load_session_error.rs index be514e138..7c2962569 100644 --- a/llama-cpp-bindings/src/context/load_session_error.rs +++ b/llama-cpp-bindings/src/context/load_session_error.rs @@ -11,7 +11,4 @@ pub enum LoadSessionError { #[error("failed to convert path {0} to str")] PathToStrError(PathBuf), - - #[error("max_length is not large enough to hold {n_out} (was {max_tokens})")] - InsufficientMaxLength { n_out: usize, max_tokens: usize }, } diff --git a/llama-cpp-bindings/src/context/kv_cache.rs b/llama-cpp-bindings/src/context/memory.rs similarity index 52% rename from llama-cpp-bindings/src/context/kv_cache.rs rename to llama-cpp-bindings/src/context/memory.rs index c346b7437..96152dcf5 100644 --- a/llama-cpp-bindings/src/context/kv_cache.rs +++ b/llama-cpp-bindings/src/context/memory.rs @@ -1,32 +1,37 @@ +use std::ffi::c_char; use std::ffi::c_int; use std::num::NonZeroU8; -use std::os::raw::c_char; use std::ptr; -use crate::context::LlamaContext; -use crate::error::{ - ClearKvCacheSeqError, CopyKvCacheSeqError, KvCacheSeqAddError, KvCacheSeqDivError, - KvCacheSeqPosMaxError, -}; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; -fn kv_cache_seq_add_status_to_result( +use crate::context::LlamaContext; +use crate::context::memory_of; +use crate::error::memory_seq_add_error::MemorySeqAddError; +use crate::error::memory_seq_cp_error::MemorySeqCpError; +use crate::error::memory_seq_div_error::MemorySeqDivError; +use crate::error::memory_seq_pos_max_error::MemorySeqPosMaxError; +use crate::error::memory_seq_rm_error::MemorySeqRmError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::unknown_rope_type::UnknownRopeType; +use crate::model::rope_type::RopeType; + +fn memory_seq_add_status_to_result( status: llama_cpp_bindings_sys::llama_rs_memory_seq_add_status, out_error: *mut c_char, -) -> Result<(), KvCacheSeqAddError> { +) -> Result<(), MemorySeqAddError> { match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE => { - Err(KvCacheSeqAddError::IncompatibleRopeType) - } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MEM => { - Err(KvCacheSeqAddError::MemoryHandleUnavailable) + Err(MemorySeqAddError::MemoryHandleUnavailable) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_ERROR_STRING_ALLOCATION_FAILED => { - Err(KvCacheSeqAddError::NotEnoughMemory) + Err(MemorySeqAddError::NotEnoughMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_OUT_OF_MEMORY => { - Err(KvCacheSeqAddError::LlamaCppOutOfMemory) + Err(MemorySeqAddError::LlamaCppOutOfMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { @@ -36,23 +41,14 @@ fn kv_cache_seq_add_status_to_result( "reported a thrown C++ exception without an error message", ) }?; - Err(KvCacheSeqAddError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_memory_seq_add", - detail: "was given a null ctx argument", - } - .into()) + Err(MemorySeqAddError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL => { - Err(crate::FfiContractError { - operation: "llama_rs_memory_seq_add", - detail: "was given a null model argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_memory_seq_add", + detail: "was given a null ctx argument", } - other => Err(crate::FfiStatusError { + .into()), + other => Err(FfiStatusError { operation: "llama_rs_memory_seq_add", code: i64::from(other), } @@ -60,23 +56,20 @@ fn kv_cache_seq_add_status_to_result( } } -fn kv_cache_seq_div_status_to_result( +fn memory_seq_div_status_to_result( status: llama_cpp_bindings_sys::llama_rs_memory_seq_div_status, out_error: *mut c_char, -) -> Result<(), KvCacheSeqDivError> { +) -> Result<(), MemorySeqDivError> { match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE => { - Err(KvCacheSeqDivError::IncompatibleRopeType) - } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MEM => { - Err(KvCacheSeqDivError::MemoryHandleUnavailable) + Err(MemorySeqDivError::MemoryHandleUnavailable) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_ERROR_STRING_ALLOCATION_FAILED => { - Err(KvCacheSeqDivError::NotEnoughMemory) + Err(MemorySeqDivError::NotEnoughMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_OUT_OF_MEMORY => { - Err(KvCacheSeqDivError::LlamaCppOutOfMemory) + Err(MemorySeqDivError::LlamaCppOutOfMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { @@ -86,23 +79,14 @@ fn kv_cache_seq_div_status_to_result( "reported a thrown C++ exception without an error message", ) }?; - Err(KvCacheSeqDivError::Reported { message }) + Err(MemorySeqDivError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_memory_seq_div", - detail: "was given a null ctx argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL => { - Err(crate::FfiContractError { - operation: "llama_rs_memory_seq_div", - detail: "was given a null model argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_memory_seq_div", + detail: "was given a null ctx argument", } - other => Err(crate::FfiStatusError { + .into()), + other => Err(FfiStatusError { operation: "llama_rs_memory_seq_div", code: i64::from(other), } @@ -110,46 +94,40 @@ fn kv_cache_seq_div_status_to_result( } } -fn kv_cache_seq_pos_max_status_to_result( +fn memory_seq_pos_max_status_to_result( status: llama_cpp_bindings_sys::llama_rs_memory_seq_pos_max_status, position: i32, - seq_id: i32, out_error: *mut c_char, -) -> Result { +) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_OK => Ok(position), - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "context pointer was null", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "context pointer was null", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_POSITION_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "output position pointer was null", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_memory_seq_pos_max", detail: "output error pointer was null", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM => { - Err(KvCacheSeqPosMaxError::MemoryHandleUnavailable) - } - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE => { - Err(KvCacheSeqPosMaxError::SequenceIdOutOfRange { seq_id }) + Err(MemorySeqPosMaxError::MemoryHandleUnavailable) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_ERROR_STRING_ALLOCATION_FAILED => { - Err(KvCacheSeqPosMaxError::NotEnoughMemory) + Err(MemorySeqPosMaxError::NotEnoughMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_OUT_OF_MEMORY => { - Err(KvCacheSeqPosMaxError::LlamaCppOutOfMemory) + Err(MemorySeqPosMaxError::LlamaCppOutOfMemory) } llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { @@ -159,9 +137,9 @@ fn kv_cache_seq_pos_max_status_to_result( "reported a thrown C++ exception without an error message", ) }?; - Err(KvCacheSeqPosMaxError::Reported { message }) + Err(MemorySeqPosMaxError::Reported { message }) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_memory_seq_pos_max", code: i64::from(other), } @@ -170,39 +148,41 @@ fn kv_cache_seq_pos_max_status_to_result( } impl LlamaContext<'_> { - fn memory_handle(&self) -> Option { - let mem = unsafe { llama_cpp_bindings_sys::llama_get_memory(self.context.as_ptr()) }; - - if mem.is_null() { None } else { Some(mem) } - } - #[must_use] pub fn has_memory(&self) -> bool { - self.memory_handle().is_some() + memory_of(self.context).is_some() + } + + fn shifts_positions_one_per_embedding(&self) -> Result { + self.model + .rope_type() + .map(RopeType::has_one_position_per_embedding) } fn required_memory_handle( &self, - ) -> Result { - self.memory_handle() - .ok_or(CopyKvCacheSeqError::MemoryHandleUnavailable) + ) -> Result { + memory_of(self.context).ok_or(MemorySeqCpError::MemoryHandleUnavailable) } /// # Errors - /// If either position exceeds [`i32::MAX`], or the context has no memory module. - pub fn copy_kv_cache_seq( + /// If either sequence id names no sequence of the context, either position exceeds + /// [`i32::MAX`], or the context has no memory module. + pub fn memory_seq_cp( &mut self, src: i32, dest: i32, p0: Option, p1: Option, - ) -> Result<(), CopyKvCacheSeqError> { + ) -> Result<(), MemorySeqCpError> { + self.limits.ensure_sequence_id(src)?; + self.limits.ensure_sequence_id(dest)?; let p0 = p0 .map_or(Ok(-1), i32::try_from) - .map_err(CopyKvCacheSeqError::P0TooLarge)?; + .map_err(MemorySeqCpError::P0TooLarge)?; let p1 = p1 .map_or(Ok(-1), i32::try_from) - .map_err(CopyKvCacheSeqError::P1TooLarge)?; + .map_err(MemorySeqCpError::P1TooLarge)?; let mem = self.required_memory_handle()?; unsafe { llama_cpp_bindings_sys::llama_memory_seq_cp(mem, src, dest, p0, p1) }; @@ -210,25 +190,33 @@ impl LlamaContext<'_> { } /// # Errors - /// If the sequence id or either position exceeds [`i32::MAX`], or llama.cpp reports - /// that the partial sequence could not be removed. A context without a memory module - /// holds no KV cache, so the removal trivially succeeds. - pub fn clear_kv_cache_seq( + /// If the sequence id or either position exceeds [`i32::MAX`], the sequence id names no + /// sequence of the context, or llama.cpp reports that the partial sequence could not be + /// removed. `None` removes from every sequence. A context without a memory module holds no + /// KV cache, so the removal trivially succeeds. + pub fn memory_seq_rm( &mut self, seq_id: Option, p0: Option, p1: Option, - ) -> Result<(), ClearKvCacheSeqError> { + ) -> Result<(), MemorySeqRmError> { let seq_id = seq_id - .map_or(Ok(-1), i32::try_from) - .map_err(ClearKvCacheSeqError::SeqIdTooLarge)?; + .map(i32::try_from) + .transpose() + .map_err(MemorySeqRmError::SeqIdTooLarge)?; + + if let Some(sequence) = seq_id { + self.limits.ensure_sequence_id(sequence)?; + } + + let seq_id = seq_id.unwrap_or(-1); let p0 = p0 .map_or(Ok(-1), i32::try_from) - .map_err(ClearKvCacheSeqError::P0TooLarge)?; + .map_err(MemorySeqRmError::P0TooLarge)?; let p1 = p1 .map_or(Ok(-1), i32::try_from) - .map_err(ClearKvCacheSeqError::P1TooLarge)?; - let Some(mem) = self.memory_handle() else { + .map_err(MemorySeqRmError::P1TooLarge)?; + let Some(mem) = memory_of(self.context) else { return Ok(()); }; @@ -236,39 +224,51 @@ impl LlamaContext<'_> { return Ok(()); } - Err(ClearKvCacheSeqError::PartialSequenceNotRemoved { seq_id, p0, p1 }) + Err(MemorySeqRmError::PartialSequenceNotRemoved { seq_id, p0, p1 }) } - pub fn clear_kv_cache(&mut self) { - let Some(mem) = self.memory_handle() else { + pub fn memory_clear(&mut self) { + let Some(mem) = memory_of(self.context) else { return; }; let clear_data_buffers = true; unsafe { llama_cpp_bindings_sys::llama_memory_clear(mem, clear_data_buffers) }; } - pub fn kv_cache_seq_keep(&mut self, seq_id: i32) { - let Some(mem) = self.memory_handle() else { - return; - }; - unsafe { llama_cpp_bindings_sys::llama_memory_seq_keep(mem, seq_id) }; + /// # Errors + /// If the sequence id names no sequence of the context. + pub fn memory_seq_keep(&mut self, seq_id: i32) -> Result<(), SequenceIdOutOfRange> { + self.limits.ensure_sequence_id(seq_id)?; + + if let Some(mem) = memory_of(self.context) { + unsafe { llama_cpp_bindings_sys::llama_memory_seq_keep(mem, seq_id) }; + } + + Ok(()) } /// # Errors - /// If either position exceeds [`i32::MAX`], or the underlying memory operation reports a failure. - pub fn kv_cache_seq_add( + /// If the sequence id names no sequence of the context, either position exceeds + /// [`i32::MAX`], or the underlying memory operation reports a failure. + pub fn memory_seq_add( &mut self, seq_id: i32, p0: Option, p1: Option, delta: i32, - ) -> Result<(), KvCacheSeqAddError> { + ) -> Result<(), MemorySeqAddError> { + self.limits.ensure_sequence_id(seq_id)?; let p0 = p0 .map_or(Ok(-1), i32::try_from) - .map_err(KvCacheSeqAddError::P0TooLarge)?; + .map_err(MemorySeqAddError::P0TooLarge)?; let p1 = p1 .map_or(Ok(-1), i32::try_from) - .map_err(KvCacheSeqAddError::P1TooLarge)?; + .map_err(MemorySeqAddError::P1TooLarge)?; + + if !self.shifts_positions_one_per_embedding()? { + return Err(MemorySeqAddError::IncompatibleRopeType); + } + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_memory_seq_add( @@ -280,24 +280,31 @@ impl LlamaContext<'_> { &raw mut out_error, ) }; - kv_cache_seq_add_status_to_result(status, out_error) + memory_seq_add_status_to_result(status, out_error) } /// # Errors - /// If either position exceeds [`i32::MAX`], or the underlying memory operation reports a failure. - pub fn kv_cache_seq_div( + /// If the sequence id names no sequence of the context, either position exceeds + /// [`i32::MAX`], or the underlying memory operation reports a failure. + pub fn memory_seq_div( &mut self, seq_id: i32, p0: Option, p1: Option, divisor: NonZeroU8, - ) -> Result<(), KvCacheSeqDivError> { + ) -> Result<(), MemorySeqDivError> { + self.limits.ensure_sequence_id(seq_id)?; let p0 = p0 .map_or(Ok(-1), i32::try_from) - .map_err(KvCacheSeqDivError::P0TooLarge)?; + .map_err(MemorySeqDivError::P0TooLarge)?; let p1 = p1 .map_or(Ok(-1), i32::try_from) - .map_err(KvCacheSeqDivError::P1TooLarge)?; + .map_err(MemorySeqDivError::P1TooLarge)?; + + if !self.shifts_positions_one_per_embedding()? { + return Err(MemorySeqDivError::IncompatibleRopeType); + } + let divisor = c_int::from(divisor.get()); let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -310,13 +317,15 @@ impl LlamaContext<'_> { &raw mut out_error, ) }; - kv_cache_seq_div_status_to_result(status, out_error) + memory_seq_div_status_to_result(status, out_error) } /// # Errors /// - /// Returns [`KvCacheSeqPosMaxError`] if the sequence does not exist or the memory lookup fails. - pub fn kv_cache_seq_pos_max(&self, seq_id: i32) -> Result { + /// Returns [`MemorySeqPosMaxError`] if the sequence does not exist or the memory lookup fails. + pub fn memory_seq_pos_max(&self, seq_id: i32) -> Result { + self.limits.ensure_sequence_id(seq_id)?; + let mut position = -1; let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -328,82 +337,92 @@ impl LlamaContext<'_> { ) }; - kv_cache_seq_pos_max_status_to_result(status, position, seq_id, out_error) + memory_seq_pos_max_status_to_result(status, position, out_error) } } #[cfg(test)] -mod kv_cache_seq_add_status_to_result_tests { +mod memory_seq_add_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; - use super::kv_cache_seq_add_status_to_result; - use crate::error::KvCacheSeqAddError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; - #[test] - fn add_ok_status_maps_to_ok() { - let result = kv_cache_seq_add_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_OK, - ptr::null_mut(), - ); - - assert!(result.is_ok()); - } + use super::memory_seq_add_status_to_result; + use crate::error::memory_seq_add_error::MemorySeqAddError; #[test] - fn add_incompatible_rope_type_status_maps_to_incompatible_rope_type() { + fn add_ok_status_maps_to_ok() { assert_eq!( - kv_cache_seq_add_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_INCOMPATIBLE_ROPE_TYPE, + memory_seq_add_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_OK, ptr::null_mut(), ), - Err(KvCacheSeqAddError::IncompatibleRopeType) + Ok(()) ); } #[test] fn add_null_mem_status_maps_to_memory_handle_unavailable() { assert_eq!( - kv_cache_seq_add_status_to_result( + memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MEM, ptr::null_mut(), ), - Err(KvCacheSeqAddError::MemoryHandleUnavailable) + Err(MemorySeqAddError::MemoryHandleUnavailable) ); } #[test] fn add_allocation_failed_status_maps_to_not_enough_memory() { assert_eq!( - kv_cache_seq_add_status_to_result( + memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_ERROR_STRING_ALLOCATION_FAILED, ptr::null_mut(), ), - Err(KvCacheSeqAddError::NotEnoughMemory) + Err(MemorySeqAddError::NotEnoughMemory) ); } #[test] - fn add_llama_cpp_exception_status_without_a_message_is_a_contract_error_with_unknown_message() { + fn add_llama_cpp_exception_status_without_a_message_is_a_contract_violation() { assert_eq!( - kv_cache_seq_add_status_to_result( + memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_memory_seq_add", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } + #[test] + fn add_llama_cpp_exception_status_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"shift threw".as_ptr()) }; + + assert_eq!( + memory_seq_add_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(MemorySeqAddError::Reported { + message: "shift threw".to_owned() + }) + ); + } + #[test] fn add_unknown_status_is_preserved() { - let result = kv_cache_seq_add_status_to_result(255, ptr::null_mut()); + let result = memory_seq_add_status_to_result(255, ptr::null_mut()); assert_eq!( result, - Err(KvCacheSeqAddError::FfiStatus(crate::FfiStatusError { + Err(MemorySeqAddError::FfiStatus(FfiStatusError { operation: "llama_rs_memory_seq_add", code: 255, })) @@ -411,17 +430,17 @@ mod kv_cache_seq_add_status_to_result_tests { } #[test] - fn kv_cache_seq_add_status_to_result_maps_null_model() { - let outcome = kv_cache_seq_add_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_MODEL, + fn memory_seq_add_status_to_result_reports_a_null_ctx_arg_as_a_contract_violation() { + let outcome = memory_seq_add_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_NULL_CTX_ARG, ptr::null_mut(), ); assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_memory_seq_add", - detail: "was given a null model argument", + detail: "was given a null ctx argument", } .into() ) @@ -429,87 +448,97 @@ mod kv_cache_seq_add_status_to_result_tests { } #[test] - fn kv_cache_seq_add_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = kv_cache_seq_add_status_to_result( + fn memory_seq_add_status_to_result_reports_llama_cpp_running_out_of_memory() { + let outcome = memory_seq_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_ADD_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ); - assert_eq!(outcome.err(), Some(KvCacheSeqAddError::LlamaCppOutOfMemory)); + assert_eq!(outcome.err(), Some(MemorySeqAddError::LlamaCppOutOfMemory)); } } #[cfg(test)] -mod kv_cache_seq_div_status_to_result_tests { +mod memory_seq_div_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; - use super::kv_cache_seq_div_status_to_result; - use crate::error::KvCacheSeqDivError; - - #[test] - fn div_ok_status_maps_to_ok() { - let result = kv_cache_seq_div_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_OK, - ptr::null_mut(), - ); + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; - assert!(result.is_ok()); - } + use super::memory_seq_div_status_to_result; + use crate::error::memory_seq_div_error::MemorySeqDivError; #[test] - fn div_incompatible_rope_type_status_maps_to_incompatible_rope_type() { + fn div_ok_status_maps_to_ok() { assert_eq!( - kv_cache_seq_div_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_INCOMPATIBLE_ROPE_TYPE, + memory_seq_div_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_OK, ptr::null_mut(), ), - Err(KvCacheSeqDivError::IncompatibleRopeType) + Ok(()) ); } #[test] fn div_null_mem_status_maps_to_memory_handle_unavailable() { assert_eq!( - kv_cache_seq_div_status_to_result( + memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MEM, ptr::null_mut(), ), - Err(KvCacheSeqDivError::MemoryHandleUnavailable) + Err(MemorySeqDivError::MemoryHandleUnavailable) ); } #[test] fn div_allocation_failed_status_maps_to_not_enough_memory() { assert_eq!( - kv_cache_seq_div_status_to_result( + memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_ERROR_STRING_ALLOCATION_FAILED, ptr::null_mut(), ), - Err(KvCacheSeqDivError::NotEnoughMemory) + Err(MemorySeqDivError::NotEnoughMemory) ); } #[test] - fn div_llama_cpp_exception_status_without_a_message_is_a_contract_error_with_unknown_message() { + fn div_llama_cpp_exception_status_without_a_message_is_a_contract_violation() { assert_eq!( - kv_cache_seq_div_status_to_result( + memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_memory_seq_div", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } + #[test] + fn div_llama_cpp_exception_status_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"division threw".as_ptr()) }; + + assert_eq!( + memory_seq_div_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(MemorySeqDivError::Reported { + message: "division threw".to_owned() + }) + ); + } + #[test] fn div_unknown_status_is_preserved() { - let result = kv_cache_seq_div_status_to_result(255, ptr::null_mut()); + let result = memory_seq_div_status_to_result(255, ptr::null_mut()); assert_eq!( result, - Err(KvCacheSeqDivError::FfiStatus(crate::FfiStatusError { + Err(MemorySeqDivError::FfiStatus(FfiStatusError { operation: "llama_rs_memory_seq_div", code: 255, })) @@ -517,17 +546,17 @@ mod kv_cache_seq_div_status_to_result_tests { } #[test] - fn kv_cache_seq_div_status_to_result_maps_null_model() { - let outcome = kv_cache_seq_div_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_MODEL, + fn memory_seq_div_status_to_result_reports_a_null_ctx_arg_as_a_contract_violation() { + let outcome = memory_seq_div_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_NULL_CTX_ARG, ptr::null_mut(), ); assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_memory_seq_div", - detail: "was given a null model argument", + detail: "was given a null ctx argument", } .into() ) @@ -535,29 +564,32 @@ mod kv_cache_seq_div_status_to_result_tests { } #[test] - fn kv_cache_seq_div_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = kv_cache_seq_div_status_to_result( + fn memory_seq_div_status_to_result_reports_llama_cpp_running_out_of_memory() { + let outcome = memory_seq_div_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_DIV_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ); - assert_eq!(outcome.err(), Some(KvCacheSeqDivError::LlamaCppOutOfMemory)); + assert_eq!(outcome.err(), Some(MemorySeqDivError::LlamaCppOutOfMemory)); } } #[cfg(test)] -mod kv_cache_seq_pos_max_status_to_result_tests { +mod memory_seq_pos_max_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; - use super::kv_cache_seq_pos_max_status_to_result; - use crate::error::KvCacheSeqPosMaxError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::memory_seq_pos_max_status_to_result; + use crate::error::memory_seq_pos_max_error::MemorySeqPosMaxError; #[test] fn seq_pos_max_ok_status_returns_position() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_OK, 17, - 2, ptr::null_mut(), ), Ok(17) @@ -567,118 +599,111 @@ mod kv_cache_seq_pos_max_status_to_result_tests { #[test] fn seq_pos_max_null_context_status_is_contract_error() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_CTX_ARG, -1, - 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "context pointer was null", - } - )) + Err(MemorySeqPosMaxError::FfiContract(FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "context pointer was null", + })) ); } #[test] fn seq_pos_max_null_output_position_status_is_contract_error() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_POSITION_ARG, -1, - 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "output position pointer was null", - } - )) + Err(MemorySeqPosMaxError::FfiContract(FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "output position pointer was null", + })) ); } #[test] fn seq_pos_max_null_output_error_status_is_contract_error() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_OUT_ERROR_ARG, -1, - 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "output error pointer was null", - } - )) + Err(MemorySeqPosMaxError::FfiContract(FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "output error pointer was null", + })) ); } #[test] fn seq_pos_max_null_memory_status_maps_to_memory_handle_unavailable() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_NULL_MEM, -1, - 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::MemoryHandleUnavailable) + Err(MemorySeqPosMaxError::MemoryHandleUnavailable) ); } #[test] - fn seq_pos_max_out_of_range_status_preserves_sequence_id() { + fn seq_pos_max_allocation_failed_status_maps_to_not_enough_memory() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_SEQ_ID_OUT_OF_RANGE, + memory_seq_pos_max_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_ERROR_STRING_ALLOCATION_FAILED, -1, - 27, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::SequenceIdOutOfRange { seq_id: 27 }) + Err(MemorySeqPosMaxError::NotEnoughMemory) ); } #[test] - fn seq_pos_max_allocation_failed_status_maps_to_not_enough_memory() { + fn seq_pos_max_llama_cpp_exception_status_without_a_message_is_a_contract_violation() { assert_eq!( - kv_cache_seq_pos_max_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_ERROR_STRING_ALLOCATION_FAILED, + memory_seq_pos_max_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, -1, - 2, ptr::null_mut(), ), - Err(KvCacheSeqPosMaxError::NotEnoughMemory) + Err(CppStringError::Missing(FfiContractError { + operation: "llama_rs_memory_seq_pos_max", + detail: "reported a thrown C++ exception without an error message", + }) + .into()) ); } #[test] - fn seq_pos_max_llama_cpp_exception_status_without_a_message_is_a_contract_error_error() { + fn seq_pos_max_llama_cpp_exception_status_reports_its_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"position lookup threw".as_ptr()) + }; + assert_eq!( - kv_cache_seq_pos_max_status_to_result( + memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_THREW_CXX_EXCEPTION, -1, - 2, - ptr::null_mut(), + out_error, ), - Err(crate::FfiContractError { - operation: "llama_rs_memory_seq_pos_max", - detail: "reported a thrown C++ exception without an error message", - } - .into()) + Err(MemorySeqPosMaxError::Reported { + message: "position lookup threw".to_owned() + }) ); } #[test] fn seq_pos_max_unknown_status_is_preserved() { assert_eq!( - kv_cache_seq_pos_max_status_to_result(255, -1, 2, ptr::null_mut(),), - Err(KvCacheSeqPosMaxError::FfiStatus(crate::FfiStatusError { + memory_seq_pos_max_status_to_result(255, -1, ptr::null_mut(),), + Err(MemorySeqPosMaxError::FfiStatus(FfiStatusError { operation: "llama_rs_memory_seq_pos_max", code: 255, })) @@ -686,16 +711,15 @@ mod kv_cache_seq_pos_max_status_to_result_tests { } #[test] - fn kv_cache_seq_pos_max_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = kv_cache_seq_pos_max_status_to_result( + fn memory_seq_pos_max_status_to_result_reports_llama_cpp_running_out_of_memory() { + let outcome = memory_seq_pos_max_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MEMORY_SEQ_POS_MAX_LLAMA_CPP_OUT_OF_MEMORY, 0, - 0, ptr::null_mut(), ); assert_eq!( outcome.err(), - Some(KvCacheSeqPosMaxError::LlamaCppOutOfMemory) + Some(MemorySeqPosMaxError::LlamaCppOutOfMemory) ); } } diff --git a/llama-cpp-bindings/src/context/params.rs b/llama-cpp-bindings/src/context/params.rs index 1538dc16c..462c1beaa 100644 --- a/llama-cpp-bindings/src/context/params.rs +++ b/llama-cpp-bindings/src/context/params.rs @@ -1,18 +1,16 @@ +use std::ffi::c_void; use std::fmt::Debug; use std::num::NonZeroU32; -pub use crate::context::kv_cache_type::KvCacheType; -pub use crate::context::llama_attention_type::LlamaAttentionType; -pub use crate::context::llama_pooling_type::LlamaPoolingType; -pub use crate::context::rope_scaling_type::RopeScalingType; +use crate::context::kv_cache_type::KvCacheType; +use crate::context::llama_attention_type::LlamaAttentionType; +use crate::context::llama_pooling_type::LlamaPoolingType; +use crate::context::rope_scaling_type::RopeScalingType; +use crate::error::unknown_attention_type::UnknownAttentionType; +use crate::error::unknown_pooling_type::UnknownPoolingType; +use crate::error::unknown_rope_scaling_type::UnknownRopeScalingType; #[derive(Debug, Clone, Copy)] -#[expect( - missing_docs, - reason = "field meanings mirror llama.cpp's `llama_context_params` C struct; restating each \ - one inline would risk drift from the upstream spec — the doc-comment on the struct \ - points at the canonical reference" -)] pub struct LlamaContextParams { pub context_params: llama_cpp_bindings_sys::llama_context_params, } @@ -85,9 +83,12 @@ impl LlamaContextParams { self } - #[must_use] - pub fn rope_scaling_type(&self) -> RopeScalingType { - RopeScalingType::from(self.context_params.rope_scaling_type) + /// # Errors + /// + /// Returns [`UnknownRopeScalingType`] when the raw params hold a value llama.cpp does not + /// define. + pub fn rope_scaling_type(&self) -> Result { + RopeScalingType::try_from(self.context_params.rope_scaling_type) } #[must_use] @@ -155,10 +156,7 @@ impl LlamaContextParams { } #[must_use] - pub const fn with_cb_eval_user_data( - mut self, - cb_eval_user_data: *mut std::ffi::c_void, - ) -> Self { + pub const fn with_cb_eval_user_data(mut self, cb_eval_user_data: *mut c_void) -> Self { self.context_params.cb_eval_user_data = cb_eval_user_data; self } @@ -169,9 +167,11 @@ impl LlamaContextParams { self } - #[must_use] - pub fn pooling_type(&self) -> LlamaPoolingType { - LlamaPoolingType::from(self.context_params.pooling_type) + /// # Errors + /// + /// Returns [`UnknownPoolingType`] when the raw params hold a value llama.cpp does not define. + pub fn pooling_type(&self) -> Result { + LlamaPoolingType::try_from(self.context_params.pooling_type) } #[must_use] @@ -217,15 +217,31 @@ impl LlamaContextParams { KvCacheType::from(self.context_params.type_v) } + /// Bytes of device memory caching the experts kept in host memory, split among the devices + /// like the layers; `0` disables the cache. + #[must_use] + pub const fn with_moe_cache_size(mut self, moe_cache_size: usize) -> Self { + self.context_params.moe_cache_size = moe_cache_size; + self + } + + #[must_use] + pub const fn moe_cache_size(&self) -> usize { + self.context_params.moe_cache_size + } + #[must_use] pub fn with_attention_type(mut self, attention_type: LlamaAttentionType) -> Self { self.context_params.attention_type = i32::from(attention_type); self } - #[must_use] - pub fn attention_type(&self) -> LlamaAttentionType { - LlamaAttentionType::from(self.context_params.attention_type) + /// # Errors + /// + /// Returns [`UnknownAttentionType`] when the raw params hold a value llama.cpp does not + /// define. + pub fn attention_type(&self) -> Result { + LlamaAttentionType::try_from(self.context_params.attention_type) } #[must_use] @@ -283,17 +299,6 @@ impl LlamaContextParams { self.context_params.yarn_orig_ctx } - #[must_use] - pub const fn with_no_perf(mut self, no_perf: bool) -> Self { - self.context_params.no_perf = no_perf; - self - } - - #[must_use] - pub const fn no_perf(&self) -> bool { - self.context_params.no_perf - } - #[must_use] pub const fn with_op_offload(mut self, op_offload: bool) -> Self { self.context_params.op_offload = op_offload; @@ -326,29 +331,35 @@ impl Default for LlamaContextParams { #[cfg(test)] mod tests { - use super::{KvCacheType, LlamaAttentionType, LlamaPoolingType, RopeScalingType}; + use std::ffi::c_void; + use std::num::NonZeroU32; + use std::ptr; + + use super::KvCacheType; + use super::LlamaAttentionType; + use super::LlamaPoolingType; + use super::RopeScalingType; #[test] - fn default_params_have_expected_values() { + fn default_params_match_the_llama_cpp_context_defaults() { let params = super::LlamaContextParams::default(); - assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(512)); + assert_eq!(params.n_ctx(), NonZeroU32::new(512)); assert_eq!(params.n_batch(), 2048); assert_eq!(params.n_ubatch(), 512); - assert_eq!(params.rope_scaling_type(), RopeScalingType::Unspecified); - assert_eq!(params.pooling_type(), LlamaPoolingType::Unspecified); + assert_eq!(params.rope_scaling_type(), Ok(RopeScalingType::Unspecified)); + assert_eq!(params.pooling_type(), Ok(LlamaPoolingType::Unspecified)); } #[test] fn with_n_ctx_sets_value() { - let params = - super::LlamaContextParams::default().with_n_ctx(std::num::NonZeroU32::new(2048)); + let params = super::LlamaContextParams::default().with_n_ctx(NonZeroU32::new(2048)); - assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(2048)); + assert_eq!(params.n_ctx(), NonZeroU32::new(2048)); } #[test] - fn with_n_ctx_none_sets_zero() { + fn with_n_ctx_none_reads_back_as_no_context_size() { let params = super::LlamaContextParams::default().with_n_ctx(None); assert_eq!(params.n_ctx(), None); @@ -384,7 +395,9 @@ mod tests { #[test] fn with_embeddings_disables() { - let params = super::LlamaContextParams::default().with_embeddings(false); + let params = super::LlamaContextParams::default() + .with_embeddings(true) + .with_embeddings(false); assert!(!params.embeddings()); } @@ -398,7 +411,9 @@ mod tests { #[test] fn with_offload_kqv_enables() { - let params = super::LlamaContextParams::default().with_offload_kqv(true); + let params = super::LlamaContextParams::default() + .with_offload_kqv(false) + .with_offload_kqv(true); assert!(params.offload_kqv()); } @@ -412,33 +427,29 @@ mod tests { #[test] fn with_swa_full_enables() { - let params = super::LlamaContextParams::default().with_swa_full(true); + let params = super::LlamaContextParams::default() + .with_swa_full(false) + .with_swa_full(true); assert!(params.swa_full()); } #[test] - fn with_rope_scaling_type_linear() { - let params = - super::LlamaContextParams::default().with_rope_scaling_type(RopeScalingType::Linear); + fn with_rope_scaling_type_sets_each_scaling_type() { + let scaling_types = [ + RopeScalingType::Linear, + RopeScalingType::Yarn, + RopeScalingType::None, + ]; - assert_eq!(params.rope_scaling_type(), RopeScalingType::Linear); - } - - #[test] - fn with_rope_scaling_type_yarn() { - let params = - super::LlamaContextParams::default().with_rope_scaling_type(RopeScalingType::Yarn); - - assert_eq!(params.rope_scaling_type(), RopeScalingType::Yarn); - } - - #[test] - fn with_rope_scaling_type_none() { - let params = - super::LlamaContextParams::default().with_rope_scaling_type(RopeScalingType::None); - - assert_eq!(params.rope_scaling_type(), RopeScalingType::None); + assert_eq!( + scaling_types.map(|scaling_type| { + super::LlamaContextParams::default() + .with_rope_scaling_type(scaling_type) + .rope_scaling_type() + }), + scaling_types.map(Ok) + ); } #[test] @@ -470,69 +481,67 @@ mod tests { } #[test] - fn with_pooling_type_mean() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Mean); - - assert_eq!(params.pooling_type(), LlamaPoolingType::Mean); - } - - #[test] - fn with_pooling_type_cls() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Cls); + fn with_pooling_type_sets_each_pooling_type() { + let pooling_types = [ + LlamaPoolingType::Mean, + LlamaPoolingType::Cls, + LlamaPoolingType::Last, + LlamaPoolingType::Rank, + LlamaPoolingType::None, + ]; - assert_eq!(params.pooling_type(), LlamaPoolingType::Cls); - } - - #[test] - fn with_pooling_type_last() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Last); - - assert_eq!(params.pooling_type(), LlamaPoolingType::Last); - } - - #[test] - fn with_pooling_type_rank() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Rank); - - assert_eq!(params.pooling_type(), LlamaPoolingType::Rank); + assert_eq!( + pooling_types.map(|pooling_type| { + super::LlamaContextParams::default() + .with_pooling_type(pooling_type) + .pooling_type() + }), + pooling_types.map(Ok) + ); } #[test] - fn with_pooling_type_none() { - let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::None); + fn with_type_k_sets_value() { + let params = super::LlamaContextParams::default().with_type_k(KvCacheType::Q40); - assert_eq!(params.pooling_type(), LlamaPoolingType::None); + assert_eq!(params.type_k(), KvCacheType::Q40); } #[test] - fn with_type_k_sets_value() { - let params = super::LlamaContextParams::default().with_type_k(KvCacheType::Q4_0); + fn with_type_v_sets_value() { + let params = super::LlamaContextParams::default().with_type_v(KvCacheType::Q41); - assert_eq!(params.type_k(), KvCacheType::Q4_0); + assert_eq!(params.type_v(), KvCacheType::Q41); } #[test] - fn with_type_v_sets_value() { - let params = super::LlamaContextParams::default().with_type_v(KvCacheType::Q4_1); + fn with_moe_cache_size_sets_value() { + let params = super::LlamaContextParams::default().with_moe_cache_size(1 << 30); - assert_eq!(params.type_v(), KvCacheType::Q4_1); + assert_eq!(params.moe_cache_size(), 1 << 30); } #[test] - fn with_flash_attention_policy_sets_value() { - let params = super::LlamaContextParams::default() - .with_flash_attention_policy(llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_ENABLED); + fn with_flash_attention_policy_sets_each_policy() { + let policies = [ + llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_ENABLED, + llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_DISABLED, + ]; assert_eq!( - params.flash_attention_policy(), - llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_ENABLED + policies.map(|policy| { + super::LlamaContextParams::default() + .with_flash_attention_policy(policy) + .flash_attention_policy() + }), + policies ); } #[test] fn builder_chaining_preserves_all_values() { let params = super::LlamaContextParams::default() - .with_n_ctx(std::num::NonZeroU32::new(1024)) + .with_n_ctx(NonZeroU32::new(1024)) .with_n_batch(4096) .with_n_ubatch(256) .with_n_threads(8) @@ -543,73 +552,62 @@ mod tests { .with_rope_freq_base(5000.0) .with_rope_freq_scale(0.25); - assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(1024)); + assert_eq!(params.n_ctx(), NonZeroU32::new(1024)); assert_eq!(params.n_batch(), 4096); assert_eq!(params.n_ubatch(), 256); assert_eq!(params.n_threads(), 8); assert_eq!(params.n_threads_batch(), 12); assert!(params.embeddings()); assert!(!params.offload_kqv()); - assert_eq!(params.rope_scaling_type(), RopeScalingType::Yarn); + assert_eq!(params.rope_scaling_type(), Ok(RopeScalingType::Yarn)); assert!((params.rope_freq_base() - 5000.0).abs() < f32::EPSILON); assert!((params.rope_freq_scale() - 0.25).abs() < f32::EPSILON); } #[test] - fn with_cb_eval_sets_callback() { - extern "C" fn test_cb_eval( + fn with_cb_eval_installs_the_given_callback() { + extern "C" fn echo_whether_asked( _tensor: *mut llama_cpp_bindings_sys::ggml_tensor, - _ask: bool, - _user_data: *mut std::ffi::c_void, + ask: bool, + _user_data: *mut c_void, ) -> bool { - false + ask } - let result = test_cb_eval(std::ptr::null_mut(), false, std::ptr::null_mut()); + let params = super::LlamaContextParams::default().with_cb_eval(Some(echo_whether_asked)); - assert!(!result); - - let params = super::LlamaContextParams::default().with_cb_eval(Some(test_cb_eval)); - - assert!(params.context_params.cb_eval.is_some()); + assert_eq!( + params + .context_params + .cb_eval + .map(|cb_eval| unsafe { cb_eval(ptr::null_mut(), true, ptr::null_mut()) }), + Some(true) + ); } #[test] fn with_cb_eval_user_data_sets_pointer() { let mut value: i32 = 42; - let user_data = (&raw mut value).cast::(); + let user_data = (&raw mut value).cast::(); let params = super::LlamaContextParams::default().with_cb_eval_user_data(user_data); assert_eq!(params.context_params.cb_eval_user_data, user_data); } #[test] - fn with_flash_attention_policy_disabled() { - let params = super::LlamaContextParams::default() - .with_flash_attention_policy(llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_DISABLED); + fn with_attention_type_sets_each_attention_type() { + let attention_types = [LlamaAttentionType::Causal, LlamaAttentionType::NonCausal]; assert_eq!( - params.flash_attention_policy(), - llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_DISABLED + attention_types.map(|attention_type| { + super::LlamaContextParams::default() + .with_attention_type(attention_type) + .attention_type() + }), + attention_types.map(Ok) ); } - #[test] - fn with_attention_type_causal() { - let params = - super::LlamaContextParams::default().with_attention_type(LlamaAttentionType::Causal); - - assert_eq!(params.attention_type(), LlamaAttentionType::Causal); - } - - #[test] - fn with_attention_type_non_causal() { - let params = - super::LlamaContextParams::default().with_attention_type(LlamaAttentionType::NonCausal); - - assert_eq!(params.attention_type(), LlamaAttentionType::NonCausal); - } - #[test] fn with_yarn_ext_factor_sets_value() { let params = super::LlamaContextParams::default().with_yarn_ext_factor(1.5); @@ -645,23 +643,11 @@ mod tests { assert_eq!(params.yarn_orig_ctx(), 4096); } - #[test] - fn with_no_perf_enables() { - let params = super::LlamaContextParams::default().with_no_perf(true); - - assert!(params.no_perf()); - } - - #[test] - fn with_no_perf_disables() { - let params = super::LlamaContextParams::default().with_no_perf(false); - - assert!(!params.no_perf()); - } - #[test] fn with_op_offload_enables() { - let params = super::LlamaContextParams::default().with_op_offload(true); + let params = super::LlamaContextParams::default() + .with_op_offload(false) + .with_op_offload(true); assert!(params.op_offload()); } @@ -682,7 +668,9 @@ mod tests { #[test] fn with_kv_unified_disables() { - let params = super::LlamaContextParams::default().with_kv_unified(false); + let params = super::LlamaContextParams::default() + .with_kv_unified(true) + .with_kv_unified(false); assert!(!params.kv_unified()); } diff --git a/llama-cpp-bindings/src/context/rope_scaling_type.rs b/llama-cpp-bindings/src/context/rope_scaling_type.rs index 92aca372e..99edbcf38 100644 --- a/llama-cpp-bindings/src/context/rope_scaling_type.rs +++ b/llama-cpp-bindings/src/context/rope_scaling_type.rs @@ -1,30 +1,41 @@ -#[repr(i8)] +use crate::error::unknown_rope_scaling_type::UnknownRopeScalingType; + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum RopeScalingType { - Unspecified = -1, - None = 0, - Linear = 1, - Yarn = 2, + Unspecified, + None, + Linear, + Yarn, + LongRope, } -impl From for RopeScalingType { - fn from(value: i32) -> Self { +impl TryFrom for RopeScalingType { + type Error = UnknownRopeScalingType; + + fn try_from( + value: llama_cpp_bindings_sys::llama_rope_scaling_type, + ) -> Result { match value { - 0 => Self::None, - 1 => Self::Linear, - 2 => Self::Yarn, - _ => Self::Unspecified, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED => Ok(Self::Unspecified), + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_NONE => Ok(Self::None), + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LINEAR => Ok(Self::Linear), + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_YARN => Ok(Self::Yarn), + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LONGROPE => Ok(Self::LongRope), + unknown => Err(UnknownRopeScalingType(unknown)), } } } -impl From for i32 { +impl From for llama_cpp_bindings_sys::llama_rope_scaling_type { fn from(value: RopeScalingType) -> Self { match value { - RopeScalingType::None => 0, - RopeScalingType::Linear => 1, - RopeScalingType::Yarn => 2, - RopeScalingType::Unspecified => -1, + RopeScalingType::Unspecified => { + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED + } + RopeScalingType::None => llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_NONE, + RopeScalingType::Linear => llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LINEAR, + RopeScalingType::Yarn => llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_YARN, + RopeScalingType::LongRope => llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LONGROPE, } } } @@ -32,26 +43,46 @@ impl From for i32 { #[cfg(test)] mod tests { use super::RopeScalingType; + use crate::error::unknown_rope_scaling_type::UnknownRopeScalingType; #[test] - fn rope_scaling_type_unknown_defaults_to_unspecified() { - assert_eq!(RopeScalingType::from(99), RopeScalingType::Unspecified); - assert_eq!(RopeScalingType::from(-100), RopeScalingType::Unspecified); + fn every_llama_cpp_value_converts_to_its_variant_and_back() { + let raw_values = [ + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_NONE, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LINEAR, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_YARN, + llama_cpp_bindings_sys::LLAMA_ROPE_SCALING_TYPE_LONGROPE, + ]; + + assert_eq!( + raw_values.map(RopeScalingType::try_from), + [ + Ok(RopeScalingType::Unspecified), + Ok(RopeScalingType::None), + Ok(RopeScalingType::Linear), + Ok(RopeScalingType::Yarn), + Ok(RopeScalingType::LongRope), + ] + ); + assert_eq!( + [ + RopeScalingType::Unspecified, + RopeScalingType::None, + RopeScalingType::Linear, + RopeScalingType::Yarn, + RopeScalingType::LongRope, + ] + .map(llama_cpp_bindings_sys::llama_rope_scaling_type::from), + raw_values + ); } #[test] - fn rope_scaling_type_roundtrip_all_variants() { - for (raw, expected) in [ - (-1, RopeScalingType::Unspecified), - (0, RopeScalingType::None), - (1, RopeScalingType::Linear), - (2, RopeScalingType::Yarn), - ] { - let from_raw = RopeScalingType::from(raw); - assert_eq!(from_raw, expected); - - let back_to_raw: i32 = from_raw.into(); - assert_eq!(back_to_raw, raw); - } + fn a_value_llama_cpp_does_not_define_is_refused() { + assert_eq!( + RopeScalingType::try_from(99), + Err(UnknownRopeScalingType(99)) + ); } } diff --git a/llama-cpp-bindings/src/context/sequence_output_capacity.rs b/llama-cpp-bindings/src/context/sequence_output_capacity.rs index 5192bf13d..592ae1a8d 100644 --- a/llama-cpp-bindings/src/context/sequence_output_capacity.rs +++ b/llama-cpp-bindings/src/context/sequence_output_capacity.rs @@ -1,5 +1,5 @@ -use crate::LlamaContextLoadError; -use crate::context::params::LlamaAttentionType; +use crate::context::llama_attention_type::LlamaAttentionType; +use crate::error::llama_context_load_error::LlamaContextLoadError; pub struct SequenceOutputCapacity { pub attention_type: LlamaAttentionType, @@ -53,8 +53,8 @@ impl SequenceOutputCapacity { #[cfg(test)] mod tests { use super::SequenceOutputCapacity; - use crate::LlamaContextLoadError; - use crate::context::params::LlamaAttentionType; + use crate::context::llama_attention_type::LlamaAttentionType; + use crate::error::llama_context_load_error::LlamaContextLoadError; const TWO_SEQUENCES_PAST_ONE_OUTPUT: Result<(), LlamaContextLoadError> = Err(LlamaContextLoadError::SequencesExceedOutputCapacity { diff --git a/llama-cpp-bindings/src/context/session.rs b/llama-cpp-bindings/src/context/session.rs index 7e30dc05a..836eec1a0 100644 --- a/llama-cpp-bindings/src/context/session.rs +++ b/llama-cpp-bindings/src/context/session.rs @@ -1,5 +1,11 @@ use std::ffi::CString; +use std::ffi::c_char; use std::path::Path; +use std::ptr; + +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::context::LlamaContext; use crate::context::llama_state_seq_flags::LlamaStateSeqFlags; @@ -14,37 +20,33 @@ use crate::token::LlamaToken; fn state_data_status_to_result( status: llama_cpp_bindings_sys::llama_rs_state_data_status, byte_count: usize, - out_error: *mut std::ffi::c_char, + out_error: *mut c_char, operation: &'static str, ) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_OK => Ok(byte_count), - llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_CTX_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_CTX_ARG => Err(FfiContractError { operation, detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_BUFFER_ARG => { - Err(crate::FfiContractError { - operation, - detail: "was given a null buffer argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_BUFFER_ARG => Err(FfiContractError { + operation, + detail: "was given a null buffer argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_BYTE_COUNT_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation, detail: "was given a null out_byte_count argument", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation, - detail: "was given a null out_error argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_ERROR_ARG => Err(FfiContractError { + operation, + detail: "was given a null out_error argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_ERROR_STRING_ALLOCATION_FAILED => { Err(StateDataError::NotEnoughMemory) } @@ -53,7 +55,7 @@ fn state_data_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { - llama_cpp_ffi_status::read_and_free_cpp_string( + read_and_free_cpp_string( out_error, operation, "reported a thrown C++ exception without an error message", @@ -62,7 +64,7 @@ fn state_data_status_to_result( Err(StateDataError::Reported { message }) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation, code: i64::from(other), } @@ -70,21 +72,32 @@ fn state_data_status_to_result( } } +fn state_data_restore_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_state_data_status, + restored_bytes: usize, + provided_bytes: usize, + out_error: *mut c_char, + operation: &'static str, +) -> Result { + let restored = state_data_status_to_result(status, restored_bytes, out_error, operation)?; + + if restored == 0 && provided_bytes != 0 { + return Err(StateDataError::NothingRestored { provided_bytes }); + } + + Ok(restored) +} + fn process_session_load_result( success: bool, n_out: usize, - max_tokens: usize, mut tokens: Vec, ) -> Result, LoadSessionError> { if !success { return Err(LoadSessionError::FailedToLoad); } - if n_out > max_tokens { - return Err(LoadSessionError::InsufficientMaxLength { n_out, max_tokens }); - } - - unsafe { tokens.set_len(n_out) }; + tokens.truncate(n_out); Ok(tokens) } @@ -92,18 +105,13 @@ fn process_session_load_result( fn process_seq_load_result( bytes_read: usize, n_out: usize, - max_tokens: usize, mut tokens: Vec, ) -> Result { if bytes_read == 0 { return Err(LoadSeqStateError::FailedToLoad); } - if n_out > max_tokens { - return Err(LoadSeqStateError::InsufficientMaxLength { n_out, max_tokens }); - } - - unsafe { tokens.set_len(n_out) }; + tokens.truncate(n_out); Ok(LoadedSeqState { tokens, bytes_read }) } @@ -154,7 +162,7 @@ impl LlamaContext<'_> { .ok_or_else(|| LoadSessionError::PathToStrError(path.to_path_buf()))?; let cstr = CString::new(path)?; - let mut tokens: Vec = Vec::with_capacity(max_tokens); + let mut tokens = vec![LlamaToken::new(0); max_tokens]; let mut n_out = 0; let tokens_out = tokens @@ -170,7 +178,7 @@ impl LlamaContext<'_> { &raw mut n_out, ) }; - process_session_load_result(success, n_out, max_tokens, tokens) + process_session_load_result(success, n_out, tokens) } /// # Errors @@ -225,7 +233,7 @@ impl LlamaContext<'_> { .ok_or_else(|| LoadSeqStateError::PathToStrError(path.to_path_buf()))?; let cstr = CString::new(path)?; - let mut tokens: Vec = Vec::with_capacity(max_tokens); + let mut tokens = vec![LlamaToken::new(0); max_tokens]; let mut n_out = 0; let tokens_out = tokens @@ -243,11 +251,11 @@ impl LlamaContext<'_> { ) }; - process_seq_load_result(bytes_read, n_out, max_tokens, tokens) + process_seq_load_result(bytes_read, n_out, tokens) } #[must_use] - pub fn get_state_size(&self) -> usize { + pub fn state_get_size(&self) -> usize { unsafe { llama_cpp_bindings_sys::llama_state_get_size(self.context.as_ptr()) } } @@ -259,9 +267,9 @@ impl LlamaContext<'_> { /// # Safety /// /// The `dest` buffer must be large enough to hold the complete state data. - pub unsafe fn copy_state_data(&self, dest: &mut [u8]) -> Result { + pub unsafe fn state_get_data(&self, dest: &mut [u8]) -> Result { let mut byte_count = 0usize; - let mut out_error: *mut std::ffi::c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_state_get_data( self.context.as_ptr(), @@ -277,16 +285,16 @@ impl LlamaContext<'_> { /// # Safety /// - /// The `src` buffer must contain data previously obtained from [`copy_state_data`](Self::copy_state_data) + /// The `src` buffer must contain data previously obtained from [`state_get_data`](Self::state_get_data) /// on a compatible context (same model and parameters). Passing arbitrary or corrupted bytes /// will lead to undefined behavior. /// /// # Errors /// /// Returns [`StateDataError`] when the llama.cpp deserializer rejects the buffer. - pub unsafe fn set_state_data(&mut self, src: &[u8]) -> Result { + pub unsafe fn state_set_data(&mut self, src: &[u8]) -> Result { let mut byte_count = 0usize; - let mut out_error: *mut std::ffi::c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_state_set_data( self.context.as_ptr(), @@ -297,16 +305,13 @@ impl LlamaContext<'_> { ) }; - let restored = - state_data_status_to_result(status, byte_count, out_error, "llama_rs_state_set_data")?; - - if restored == 0 && !src.is_empty() { - return Err(StateDataError::NothingRestored { - provided_bytes: src.len(), - }); - } - - Ok(restored) + state_data_restore_status_to_result( + status, + byte_count, + src.len(), + out_error, + "llama_rs_state_set_data", + ) } #[must_use] @@ -334,7 +339,7 @@ impl LlamaContext<'_> { flags: &LlamaStateSeqFlags, ) -> Result { let mut byte_count = 0usize; - let mut out_error: *mut std::ffi::c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_state_seq_get_data( self.context.as_ptr(), @@ -365,7 +370,7 @@ impl LlamaContext<'_> { flags: &LlamaStateSeqFlags, ) -> Result { let mut byte_count = 0usize; - let mut out_error: *mut std::ffi::c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_state_seq_set_data( self.context.as_ptr(), @@ -378,20 +383,13 @@ impl LlamaContext<'_> { ) }; - let restored = state_data_status_to_result( + state_data_restore_status_to_result( status, byte_count, + src.len(), out_error, "llama_rs_state_seq_set_data", - )?; - - if restored == 0 && !src.is_empty() { - return Err(StateDataError::NothingRestored { - provided_bytes: src.len(), - }); - } - - Ok(restored) + ) } } @@ -402,84 +400,66 @@ mod process_session_load_result_tests { use crate::token::LlamaToken; #[test] - fn session_load_success_within_bounds() { - let tokens = vec![LlamaToken::new(0); 100]; - let result = process_session_load_result(true, 10, 100, tokens); + fn session_load_truncates_the_tokens_to_the_loaded_count() { + let tokens: Vec = (0..100).map(LlamaToken::new).collect(); - assert!(result.is_ok()); - assert_eq!(result.unwrap().len(), 10); + assert_eq!( + process_session_load_result(true, 10, tokens), + Ok((0..10).map(LlamaToken::new).collect()) + ); } #[test] fn session_load_fails_when_not_successful() { let tokens = vec![LlamaToken::new(0); 100]; - let result = process_session_load_result(false, 0, 100, tokens); + let result = process_session_load_result(false, 0, tokens); assert_eq!(result, Err(LoadSessionError::FailedToLoad)); } - - #[test] - fn session_load_fails_when_n_out_exceeds_max() { - let tokens = vec![LlamaToken::new(0); 100]; - let result = process_session_load_result(true, 101, 100, tokens); - - assert_eq!( - result, - Err(LoadSessionError::InsufficientMaxLength { - n_out: 101, - max_tokens: 100, - }) - ); - } } #[cfg(test)] mod process_seq_load_result_tests { use super::process_seq_load_result; use crate::context::load_seq_state_error::LoadSeqStateError; + use crate::context::loaded_seq_state::LoadedSeqState; use crate::token::LlamaToken; #[test] - fn seq_load_success_within_bounds() { - let tokens = vec![LlamaToken::new(0); 100]; - let loaded = process_seq_load_result(42, 10, 100, tokens).expect("bounded load succeeds"); + fn seq_load_truncates_the_tokens_to_the_loaded_count_and_reports_the_bytes_read() { + let tokens: Vec = (0..100).map(LlamaToken::new).collect(); - assert_eq!(loaded.tokens.len(), 10); - assert_eq!(loaded.bytes_read, 42); + assert_eq!( + process_seq_load_result(42, 10, tokens), + Ok(LoadedSeqState { + tokens: (0..10).map(LlamaToken::new).collect(), + bytes_read: 42, + }) + ); } #[test] fn seq_load_fails_when_zero_bytes_read() { let tokens = vec![LlamaToken::new(0); 100]; - let result = process_seq_load_result(0, 0, 100, tokens); + let result = process_seq_load_result(0, 0, tokens); assert_eq!(result, Err(LoadSeqStateError::FailedToLoad)); } - - #[test] - fn seq_load_fails_when_n_out_exceeds_max() { - let tokens = vec![LlamaToken::new(0); 100]; - let result = process_seq_load_result(42, 101, 100, tokens); - - assert_eq!( - result, - Err(LoadSeqStateError::InsufficientMaxLength { - n_out: 101, - max_tokens: 100, - }) - ); - } } #[cfg(test)] mod state_data_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::state_data_status_to_result; use crate::context::state_data_error::StateDataError; #[test] - fn state_data_status_to_result_maps_llama_cpp_out_of_memory() { + fn state_data_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = state_data_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_OUT_OF_MEMORY, 0, @@ -488,4 +468,137 @@ mod state_data_status_to_result_tests { ); assert_eq!(outcome.err(), Some(StateDataError::LlamaCppOutOfMemory)); } + + #[test] + fn state_data_status_to_result_maps_each_null_argument_to_a_contract_violation_of_the_operation() + { + for (status, detail) in [ + ( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_CTX_ARG, + "was given a null ctx argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_BUFFER_ARG, + "was given a null buffer argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_BYTE_COUNT_ARG, + "was given a null out_byte_count argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_NULL_OUT_ERROR_ARG, + "was given a null out_error argument", + ), + ] { + assert_eq!( + state_data_status_to_result(status, 0, ptr::null_mut(), "llama_rs_state_get_data"), + Err(StateDataError::FfiContract(FfiContractError { + operation: "llama_rs_state_get_data", + detail, + })) + ); + } + } + + #[test] + fn state_data_status_to_result_reports_not_enough_memory_for_the_error_string() { + assert_eq!( + state_data_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_ERROR_STRING_ALLOCATION_FAILED, + 0, + ptr::null_mut(), + "llama_rs_state_get_data", + ), + Err(StateDataError::NotEnoughMemory) + ); + } + + #[test] + fn state_data_status_to_result_reports_an_exception_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"serializer threw".as_ptr()) }; + + assert_eq!( + state_data_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + "llama_rs_state_get_data", + ), + Err(StateDataError::Reported { + message: "serializer threw".to_owned() + }) + ); + } + + #[test] + fn state_data_status_to_result_maps_an_exception_without_a_message_to_a_contract_violation() { + assert_eq!( + state_data_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + ptr::null_mut(), + "llama_rs_state_get_data", + ), + Err(StateDataError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_state_get_data", + detail: "reported a thrown C++ exception without an error message", + } + ))) + ); + } + + #[test] + fn state_data_status_to_result_preserves_an_unknown_status() { + assert_eq!( + state_data_status_to_result(255, 0, ptr::null_mut(), "llama_rs_state_get_data"), + Err(StateDataError::FfiStatus(FfiStatusError { + operation: "llama_rs_state_get_data", + code: 255, + })) + ); + } +} + +#[cfg(test)] +mod state_data_restore_status_to_result_tests { + use std::ptr; + + use super::state_data_restore_status_to_result; + use crate::context::state_data_error::StateDataError; + + #[test] + fn restoring_state_data_fails_only_when_a_non_empty_snapshot_restores_nothing() { + assert_eq!( + [(0, 16), (0, 0), (16, 16)].map(|(restored_bytes, provided_bytes)| { + state_data_restore_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_OK, + restored_bytes, + provided_bytes, + ptr::null_mut(), + "llama_rs_state_seq_set_data", + ) + }), + [ + Err(StateDataError::NothingRestored { provided_bytes: 16 }), + Ok(0), + Ok(16), + ] + ); + } + + #[test] + fn restoring_state_data_propagates_a_failed_status() { + assert_eq!( + state_data_restore_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_STATE_DATA_LLAMA_CPP_OUT_OF_MEMORY, + 0, + 16, + ptr::null_mut(), + "llama_rs_state_seq_set_data", + ), + Err(StateDataError::LlamaCppOutOfMemory) + ); + } } diff --git a/llama-cpp-bindings/src/context/state_data_error.rs b/llama-cpp-bindings/src/context/state_data_error.rs index 0487b2980..10355dfe5 100644 --- a/llama-cpp-bindings/src/context/state_data_error.rs +++ b/llama-cpp-bindings/src/context/state_data_error.rs @@ -1,9 +1,15 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum StateDataError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/cpp_text_view.rs b/llama-cpp-bindings/src/cpp_text_view.rs new file mode 100644 index 000000000..92088fee1 --- /dev/null +++ b/llama-cpp-bindings/src/cpp_text_view.rs @@ -0,0 +1,20 @@ +use std::ffi::c_char; +use std::slice; +use std::str; +use std::str::Utf8Error; + +/// Borrows the UTF-8 text of a C++ string llama.cpp exposes as a pointer and a length. +/// +/// # Safety +/// +/// `text` must point to `length` readable bytes that stay alive and unchanged for `'text`. +/// +/// # Errors +/// +/// Returns [`Utf8Error`] when the bytes are not UTF-8. +pub const unsafe fn cpp_text_view<'text>( + text: *const c_char, + length: usize, +) -> Result<&'text str, Utf8Error> { + str::from_utf8(unsafe { slice::from_raw_parts(text.cast::(), length) }) +} diff --git a/llama-cpp-bindings/src/dry_sequence_breakers.rs b/llama-cpp-bindings/src/dry_sequence_breakers.rs new file mode 100644 index 000000000..7ce4ba658 --- /dev/null +++ b/llama-cpp-bindings/src/dry_sequence_breakers.rs @@ -0,0 +1,139 @@ +use std::collections::HashSet; +use std::ffi::CString; + +use llama_cpp_bindings_sys::LLAMA_DRY_MAX_SEQ_BREAKERS; + +use crate::error::sampling_error::SamplingError; + +const DRY_SEQUENCE_BREAKER_MAX_BYTES: usize = 40; + +fn validated_dry_sequence_breaker( + index: usize, + seq_breaker: &[u8], +) -> Result { + if seq_breaker.is_empty() { + return Err(SamplingError::DrySequenceBreakerEmpty { index }); + } + + if seq_breaker.len() > DRY_SEQUENCE_BREAKER_MAX_BYTES { + return Err(SamplingError::DrySequenceBreakerTooLong { + index, + length: seq_breaker.len(), + max_length: DRY_SEQUENCE_BREAKER_MAX_BYTES, + }); + } + + CString::new(seq_breaker) + .map_err(|source| SamplingError::DrySequenceBreakerContainsNul { index, source }) +} + +/// Validates DRY sequence breakers against the limits `llama_sampler_init_dry` enforces by +/// silently skipping, truncating, or ignoring breakers. +/// +/// # Errors +/// +/// Returns the [`SamplingError`] variant describing the first breaker llama.cpp would skip, +/// truncate, or reject, or [`SamplingError::TooManyDistinctDrySequenceBreakers`] when more +/// distinct breakers are given than llama.cpp uses. +pub fn dry_sequence_breakers( + seq_breakers: impl IntoIterator>, +) -> Result, SamplingError> { + let validated = seq_breakers + .into_iter() + .enumerate() + .map(|(index, seq_breaker)| validated_dry_sequence_breaker(index, seq_breaker.as_ref())) + .collect::, SamplingError>>()?; + let distinct = validated.iter().collect::>().len(); + let max = LLAMA_DRY_MAX_SEQ_BREAKERS as usize; + + if distinct > max { + return Err(SamplingError::TooManyDistinctDrySequenceBreakers { distinct, max }); + } + + Ok(validated) +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + + use super::dry_sequence_breakers; + use crate::error::sampling_error::SamplingError; + + fn distinct_breakers(count: usize) -> Vec { + (0..count) + .map(|breaker_index| format!("breaker{breaker_index}")) + .collect() + } + + #[test] + fn breakers_within_every_limit_are_passed_through_in_order() { + assert_eq!( + dry_sequence_breakers(["\n", ":"]), + Ok(vec![c"\n".to_owned(), c":".to_owned()]) + ); + } + + #[test] + fn an_empty_breaker_is_refused_instead_of_skipped() { + assert_eq!( + dry_sequence_breakers(["\n", ""]), + Err(SamplingError::DrySequenceBreakerEmpty { index: 1 }) + ); + } + + #[test] + fn a_breaker_longer_than_forty_bytes_is_refused_instead_of_truncated() { + assert_eq!( + dry_sequence_breakers(["a".repeat(41)]), + Err(SamplingError::DrySequenceBreakerTooLong { + index: 0, + length: 41, + max_length: 40, + }) + ); + } + + #[test] + fn a_breaker_of_exactly_forty_bytes_is_accepted() { + assert_eq!( + dry_sequence_breakers(["a".repeat(40)]), + Ok(vec![CString::new("a".repeat(40)).unwrap()]) + ); + } + + #[test] + fn a_breaker_with_an_interior_nul_byte_is_refused() { + let nul_error = CString::new("hello\0world").unwrap_err(); + + assert_eq!( + dry_sequence_breakers(["ok", "hello\0world"]), + Err(SamplingError::DrySequenceBreakerContainsNul { + index: 1, + source: nul_error, + }) + ); + } + + #[test] + fn sixty_four_distinct_breakers_with_duplicates_are_accepted() { + let mut breakers = distinct_breakers(64); + breakers.push("breaker0".to_owned()); + + assert_eq!( + dry_sequence_breakers(&breakers).map(|validated| validated.len()), + Ok(65) + ); + } + + #[test] + fn more_than_sixty_four_distinct_breakers_are_refused_instead_of_ignored() { + assert_eq!( + dry_sequence_breakers(distinct_breakers(65)), + Err(SamplingError::TooManyDistinctDrySequenceBreakers { + distinct: 65, + max: 64, + }) + ); + } +} diff --git a/llama-cpp-bindings/src/error.rs b/llama-cpp-bindings/src/error.rs index ce43614b0..9260a422f 100644 --- a/llama-cpp-bindings/src/error.rs +++ b/llama-cpp-bindings/src/error.rs @@ -1,8 +1,8 @@ -pub mod apply_chat_template_error; -pub mod bracketed_args_failure; +pub mod backend_device_text_error; +pub mod batch_transfer_error; +pub mod chat_session_error; pub mod chat_template_error; -pub mod clear_kv_cache_seq_error; -pub mod copy_kv_cache_seq_error; +pub mod chat_templates_error; pub mod decode_error; pub mod embeddings_error; pub mod encode_error; @@ -10,79 +10,40 @@ pub mod eval_multimodal_chunks_error; pub mod fit_error; pub mod grammar_error; pub mod grammar_runtime_error; -pub mod json_object_failure; +pub mod ingest_error; pub mod json_schema_to_grammar_error; -pub mod key_value_xml_tags_failure; -pub mod kv_cache_seq_add_error; -pub mod kv_cache_seq_div_error; -pub mod kv_cache_seq_pos_max_error; +pub mod llama_backend_device_error; pub mod llama_context_load_error; pub mod llama_cpp_error; pub mod llama_lora_adapter_init_error; pub mod llama_lora_adapters_error; pub mod llama_model_load_error; pub mod logits_error; -pub mod marker_detection_error; +pub mod memory_seq_add_error; +pub mod memory_seq_cp_error; +pub mod memory_seq_div_error; +pub mod memory_seq_pos_max_error; +pub mod memory_seq_rm_error; pub mod meta_val_error; pub mod model_params_error; pub mod new_llama_chat_message_error; -pub mod paired_quote_failure; pub mod parse_chat_message_error; +pub mod process_failure; pub mod sample_error; pub mod sampler_accept_error; pub mod sampler_apply_error; pub mod sampling_error; +pub mod sequence_id_out_of_range; pub mod string_to_token_error; pub mod token_attr_error; pub mod token_outside_vocabulary; -pub mod token_sampling_error; pub mod token_to_string_error; -pub mod tool_call_format_failure; -pub mod xml_function_tags_failure; +pub mod unknown_attention_type; +pub mod unknown_backend_device_type; +pub mod unknown_pooling_type; +pub mod unknown_rope_scaling_type; +pub mod unknown_rope_type; -pub use llama_cpp_ffi_status::FfiContractError; -pub use llama_cpp_ffi_status::FfiStatusError; - -pub use apply_chat_template_error::ApplyChatTemplateError; -pub use bracketed_args_failure::BracketedArgsFailure; -pub use chat_template_error::ChatTemplateError; -pub use clear_kv_cache_seq_error::ClearKvCacheSeqError; -pub use copy_kv_cache_seq_error::CopyKvCacheSeqError; -pub use decode_error::DecodeError; -pub use embeddings_error::EmbeddingsError; -pub use encode_error::EncodeError; -pub use eval_multimodal_chunks_error::EvalMultimodalChunksError; -pub use fit_error::FitError; -pub use grammar_error::GrammarError; -pub use grammar_runtime_error::GrammarRuntimeError; -pub use json_object_failure::JsonObjectFailure; -pub use json_schema_to_grammar_error::JsonSchemaToGrammarError; -pub use key_value_xml_tags_failure::KeyValueXmlTagsFailure; -pub use kv_cache_seq_add_error::KvCacheSeqAddError; -pub use kv_cache_seq_div_error::KvCacheSeqDivError; -pub use kv_cache_seq_pos_max_error::KvCacheSeqPosMaxError; -pub use llama_context_load_error::LlamaContextLoadError; -pub use llama_cpp_error::LlamaCppError; -pub use llama_lora_adapter_init_error::LlamaLoraAdapterInitError; -pub use llama_lora_adapters_error::LlamaLoraAdaptersError; -pub use llama_model_load_error::LlamaModelLoadError; -pub use logits_error::LogitsError; -pub use marker_detection_error::MarkerDetectionError; -pub use meta_val_error::MetaValError; -pub use model_params_error::ModelParamsError; -pub use new_llama_chat_message_error::NewLlamaChatMessageError; -pub use paired_quote_failure::PairedQuoteFailure; -pub use parse_chat_message_error::ParseChatMessageError; -pub use sample_error::SampleError; -pub use sampler_accept_error::SamplerAcceptError; -pub use sampler_apply_error::SamplerApplyError; -pub use sampling_error::SamplingError; -pub use string_to_token_error::StringToTokenError; -pub use token_attr_error::TokenAttrError; -pub use token_outside_vocabulary::TokenOutsideVocabulary; -pub use token_sampling_error::TokenSamplingError; -pub use token_to_string_error::TokenToStringError; -pub use tool_call_format_failure::ToolCallFormatFailure; -pub use xml_function_tags_failure::XmlFunctionTagsFailure; +use llama_cpp_error::LlamaCppError; pub type Result = std::result::Result; diff --git a/llama-cpp-bindings/src/error/apply_chat_template_error.rs b/llama-cpp-bindings/src/error/apply_chat_template_error.rs deleted file mode 100644 index 12a53c44e..000000000 --- a/llama-cpp-bindings/src/error/apply_chat_template_error.rs +++ /dev/null @@ -1,17 +0,0 @@ -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum ApplyChatTemplateError { - #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), - #[error("the model has no vocab")] - NoVocab, - #[error("the model's chat template rendered an empty prompt or could not be rendered")] - TemplateApplicationFailed, - #[error("not enough memory to render the chat template")] - NotEnoughMemory, - #[error("the llama.cpp chat-template renderer ran out of memory")] - LlamaCppOutOfMemory, - #[error("{message}")] - Reported { message: String }, -} diff --git a/llama-cpp-bindings/src/error/backend_device_text_error.rs b/llama-cpp-bindings/src/error/backend_device_text_error.rs new file mode 100644 index 000000000..d9d5e9741 --- /dev/null +++ b/llama-cpp-bindings/src/error/backend_device_text_error.rs @@ -0,0 +1,9 @@ +use std::str::Utf8Error; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum BackendDeviceTextError { + #[error("ggml reported no text")] + Null, + #[error("ggml reported text that is not valid UTF-8")] + NotUtf8(#[from] Utf8Error), +} diff --git a/llama-cpp-bindings/src/error/batch_transfer_error.rs b/llama-cpp-bindings/src/error/batch_transfer_error.rs new file mode 100644 index 000000000..ae766d17c --- /dev/null +++ b/llama-cpp-bindings/src/error/batch_transfer_error.rs @@ -0,0 +1,26 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::context::context_limits::ContextLimits; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum BatchTransferError { + #[error(transparent)] + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), + #[error("the batch was built for context limits {batch:?}, but the context has {context:?}")] + LimitsMismatch { + batch: ContextLimits, + context: ContextLimits, + }, + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/error/bracketed_args_failure.rs b/llama-cpp-bindings/src/error/bracketed_args_failure.rs deleted file mode 100644 index 4be4e8037..000000000 --- a/llama-cpp-bindings/src/error/bracketed_args_failure.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum BracketedArgsFailure { - #[error("tool call '{tool_name}' arguments are not valid JSON: {message}")] - InvalidJsonArguments { tool_name: String, message: String }, - #[error("tool call '{tool_name}' arguments truncated before JSON value completed")] - UnterminatedArguments { tool_name: String }, -} diff --git a/llama-cpp-bindings/src/error/chat_session_error.rs b/llama-cpp-bindings/src/error/chat_session_error.rs new file mode 100644 index 000000000..593c01b1b --- /dev/null +++ b/llama-cpp-bindings/src/error/chat_session_error.rs @@ -0,0 +1,44 @@ +use std::ffi::NulError; +use std::str::Utf8Error; + +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::error::grammar_error::GrammarError; +use crate::error::sampler_accept_error::SamplerAcceptError; +use crate::error::string_to_token_error::StringToTokenError; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum ChatSessionError { + #[error(transparent)] + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), + #[error("a chat message is not valid UTF-8")] + MessageNotUtf8(#[source] Utf8Error), + #[error("the chat request contains an interior NUL byte")] + RequestContainsNul(#[source] NulError), + #[error("the tools are not valid JSON: {message}")] + ToolsNotJson { message: String }, + #[error("the tools are not valid tool definitions: {message}")] + ToolsInvalid { message: String }, + #[error("the chat template cannot serve the request: {message}")] + TemplateCannotServeRequest { message: String }, + #[error("text the chat session holds is not valid UTF-8")] + SessionTextNotUtf8(#[source] Utf8Error), + #[error("the rendered prompt could not be tokenized: {0}")] + PromptTokenization(#[from] StringToTokenError), + #[error("the request's grammar sampler could not be built: {0}")] + Grammar(#[from] GrammarError), + #[error("the grammar sampler refused a generation-prompt token: {0}")] + GrammarPrefill(#[from] SamplerAcceptError), + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/error/chat_templates_error.rs b/llama-cpp-bindings/src/error/chat_templates_error.rs new file mode 100644 index 000000000..289c5e535 --- /dev/null +++ b/llama-cpp-bindings/src/error/chat_templates_error.rs @@ -0,0 +1,25 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum ChatTemplatesError { + #[error(transparent)] + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), + #[error("the model has no chat template")] + ModelHasNoChatTemplate, + #[error( + "an override template must not be empty, because llama.cpp reads an empty override as no override" + )] + EmptyTemplateOverride, + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/error/decode_error.rs b/llama-cpp-bindings/src/error/decode_error.rs index 0eb296a64..1d213892f 100644 --- a/llama-cpp-bindings/src/error/decode_error.rs +++ b/llama-cpp-bindings/src/error/decode_error.rs @@ -1,12 +1,13 @@ +use std::ffi::c_int; use std::num::NonZeroI32; -use std::os::raw::c_int; + +use crate::error::batch_transfer_error::BatchTransferError; +use crate::error::process_failure::ProcessFailure; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum DecodeError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + Process(#[from] ProcessFailure), #[error("no KV cache slot was available")] NoKvCacheSlot, #[error("decode aborted by callback")] @@ -19,14 +20,10 @@ pub enum DecodeError { ComputeFailed, #[error("decode returned an unknown status code: {code}")] UnknownStatus { code: c_int }, - #[error("not enough memory")] - NotEnoughMemory, - #[error("the llama.cpp library ran out of memory")] - LlamaCppOutOfMemory, #[error("decode batch of {n_tokens} tokens must fit a single micro batch of {n_ubatch} tokens")] BatchExceedsMicroBatch { n_tokens: i32, n_ubatch: u32 }, - #[error("{message}")] - Reported { message: String }, + #[error("the batch could not be handed to llama.cpp: {0}")] + BatchTransfer(#[from] BatchTransferError), } impl From for DecodeError { @@ -35,6 +32,8 @@ impl From for DecodeError { 1 => Self::NoKvCacheSlot, 2 => Self::Aborted, -1 => Self::BatchInvalid, + -2 => Self::DecodeOutOfMemory, + -3 => Self::ComputeFailed, error_code => Self::UnknownStatus { code: error_code }, } } @@ -44,7 +43,7 @@ impl From for DecodeError { mod tests { use std::num::NonZeroI32; - use super::DecodeError; + use crate::error::decode_error::DecodeError; #[test] fn no_kv_cache_slot_maps_from_code_one() { @@ -67,6 +66,14 @@ mod tests { assert_eq!(error, DecodeError::BatchInvalid); } + #[test] + fn allocation_and_compute_failures_map_from_codes_negative_two_and_three() { + assert_eq!( + [-2, -3].map(|code| DecodeError::from(NonZeroI32::new(code).expect("non-zero code"))), + [DecodeError::DecodeOutOfMemory, DecodeError::ComputeFailed] + ); + } + #[test] fn unrecognized_code_falls_through_to_unknown_status() { let error = DecodeError::from(NonZeroI32::new(42).expect("42 is non-zero")); diff --git a/llama-cpp-bindings/src/error/embeddings_error.rs b/llama-cpp-bindings/src/error/embeddings_error.rs index eddf1c1ad..e57a070ce 100644 --- a/llama-cpp-bindings/src/error/embeddings_error.rs +++ b/llama-cpp-bindings/src/error/embeddings_error.rs @@ -2,8 +2,8 @@ pub enum EmbeddingsError { #[error("Embeddings weren't enabled in the context options")] NotEnabled, - #[error("Logits were not enabled for the given token")] - LogitsNotEnabled, + #[error("token {token_index} names no output row of the last decode")] + TokenNotAnOutput { token_index: i32 }, #[error( "Sequence embeddings need a pooling context, but this context uses LLAMA_POOLING_TYPE_NONE" )] @@ -16,36 +16,8 @@ pub enum EmbeddingsError { "Token embeddings need a context with LLAMA_POOLING_TYPE_NONE, but it pools with {pooling_type:?}" )] TokenEmbeddingsRequireNonePooling { - pooling_type: crate::context::params::LlamaPoolingType, + pooling_type: crate::context::llama_pooling_type::LlamaPoolingType, }, #[error("Invalid embedding dimension: {0}")] InvalidEmbeddingDimension(#[source] std::num::TryFromIntError), - #[error( - "NextN embeddings need a context with LLAMA_POOLING_TYPE_NONE, but it pools with {pooling_type:?}" - )] - NextnEmbeddingsRequireNonePooling { - pooling_type: crate::context::params::LlamaPoolingType, - }, - #[error("NextN embeddings weren't enabled on this context")] - NextnEmbeddingsNotEnabled, - #[error( - "NextN embeddings are unavailable because this model's graph produces no NextN hidden state" - )] - NextnEmbeddingsUnsupported, - #[error("llama.cpp failed to reserve the graph that checks for a NextN hidden state")] - NextnProbeGraphReservationFailed, - #[error("not enough memory to report why the NextN hidden state check failed")] - NextnProbeNotEnoughMemory, - #[error("llama.cpp ran out of memory while checking for a NextN hidden state")] - NextnProbeLlamaCppOutOfMemory, - #[error("llama.cpp failed while checking for a NextN hidden state: {message}")] - NextnProbeReported { message: String }, - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), - #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error( - "No NextN embedding exists for token {token_index}; it was not marked as an output of the last decoded batch" - )] - NextnEmbeddingUnavailable { token_index: i32 }, } diff --git a/llama-cpp-bindings/src/error/encode_error.rs b/llama-cpp-bindings/src/error/encode_error.rs index 3238c040d..2c868d13c 100644 --- a/llama-cpp-bindings/src/error/encode_error.rs +++ b/llama-cpp-bindings/src/error/encode_error.rs @@ -1,12 +1,13 @@ +use std::ffi::c_int; use std::num::NonZeroI32; -use std::os::raw::c_int; + +use crate::error::batch_transfer_error::BatchTransferError; +use crate::error::process_failure::ProcessFailure; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum EncodeError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + Process(#[from] ProcessFailure), #[error("model has no encoder")] ModelHasNoEncoder, #[error("no KV cache slot was available")] @@ -19,14 +20,10 @@ pub enum EncodeError { ComputeFailed, #[error("encode returned an unknown status code: {code}")] UnknownStatus { code: c_int }, - #[error("not enough memory")] - NotEnoughMemory, - #[error("the llama.cpp library ran out of memory")] - LlamaCppOutOfMemory, #[error("encode batch of {n_tokens} tokens must fit a single micro batch of {n_ubatch} tokens")] BatchExceedsMicroBatch { n_tokens: i32, n_ubatch: u32 }, - #[error("{message}")] - Reported { message: String }, + #[error("the batch could not be handed to llama.cpp: {0}")] + BatchTransfer(#[from] BatchTransferError), } impl From for EncodeError { @@ -34,6 +31,8 @@ impl From for EncodeError { match value.get() { 1 => Self::NoKvCacheSlot, -1 => Self::BatchInvalid, + -2 => Self::EncodeOutOfMemory, + -3 => Self::ComputeFailed, error_code => Self::UnknownStatus { code: error_code }, } } @@ -43,7 +42,7 @@ impl From for EncodeError { mod tests { use std::num::NonZeroI32; - use super::EncodeError; + use crate::error::encode_error::EncodeError; #[test] fn no_kv_cache_slot_maps_from_code_one() { @@ -59,6 +58,14 @@ mod tests { assert_eq!(error, EncodeError::BatchInvalid); } + #[test] + fn allocation_and_compute_failures_map_from_codes_negative_two_and_three() { + assert_eq!( + [-2, -3].map(|code| EncodeError::from(NonZeroI32::new(code).expect("non-zero code"))), + [EncodeError::EncodeOutOfMemory, EncodeError::ComputeFailed] + ); + } + #[test] fn unrecognized_code_falls_through_to_unknown_status() { let error = EncodeError::from(NonZeroI32::new(99).expect("99 is non-zero")); diff --git a/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs b/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs index 7874616f1..24d6c5aa5 100644 --- a/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs +++ b/llama-cpp-bindings/src/error/eval_multimodal_chunks_error.rs @@ -1,6 +1,6 @@ -use crate::mtmd::MtmdEvalError; +use crate::mtmd::mtmd_eval_error::MtmdEvalError; -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum EvalMultimodalChunksError { #[error("{0}")] EvalFailed(#[from] MtmdEvalError), diff --git a/llama-cpp-bindings/src/error/fit_error.rs b/llama-cpp-bindings/src/error/fit_error.rs index 8aa5a00e9..c6a61377b 100644 --- a/llama-cpp-bindings/src/error/fit_error.rs +++ b/llama-cpp-bindings/src/error/fit_error.rs @@ -1,13 +1,21 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum FitError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), #[error("no parameter combination fits available memory")] NoFittingMemoryLayout, #[error("parameter fitting aborted")] Aborted, + #[error("{actual} memory margins were given, one per device needs exactly {expected}")] + MarginsDeviceCountMismatch { expected: usize, actual: usize }, #[error("parameter fitting returned an unknown status code: {code}")] UnknownStatus { code: i32 }, #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/error/grammar_error.rs b/llama-cpp-bindings/src/error/grammar_error.rs index e092d0053..9a67fc7f2 100644 --- a/llama-cpp-bindings/src/error/grammar_error.rs +++ b/llama-cpp-bindings/src/error/grammar_error.rs @@ -1,14 +1,20 @@ use std::ffi::NulError; -use crate::SamplingError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::error::sampling_error::SamplingError; use crate::error::token_to_string_error::TokenToStringError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum GrammarError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("sampler initialization failed: {0}")] SamplerInitialization(#[from] SamplingError), #[error("the approximate token environment could not be built: {0}")] @@ -23,8 +29,6 @@ pub enum GrammarError { RootContainsNul(#[source] NulError), #[error("a lazy-grammar trigger pattern contains an interior NUL byte")] TriggerPatternContainsNul(#[source] NulError), - #[error("a DRY sequence breaker contains an interior NUL byte")] - SequenceBreakerContainsNul(#[source] NulError), #[error("the llguidance parser factory could not be created: {message}")] LlguidanceFactoryUnavailable { message: String }, #[error("the llguidance grammar could not be parsed: {message}")] diff --git a/llama-cpp-bindings/src/error/grammar_runtime_error.rs b/llama-cpp-bindings/src/error/grammar_runtime_error.rs index 9283cbc6b..068ab90da 100644 --- a/llama-cpp-bindings/src/error/grammar_runtime_error.rs +++ b/llama-cpp-bindings/src/error/grammar_runtime_error.rs @@ -1,5 +1,7 @@ #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum GrammarRuntimeError { + #[error("the grammar parser failed a step without stopping: {message}")] + FailedWithoutStopping { message: String }, #[error("the grammar parser reached an internal error state: {message}")] InternalParserError { message: String }, #[error("the grammar lexer became too complex: {message}")] diff --git a/llama-cpp-bindings/src/error/ingest_error.rs b/llama-cpp-bindings/src/error/ingest_error.rs new file mode 100644 index 000000000..b380f2bd9 --- /dev/null +++ b/llama-cpp-bindings/src/error/ingest_error.rs @@ -0,0 +1,10 @@ +use crate::error::parse_chat_message_error::ParseChatMessageError; +use crate::error::token_to_string_error::TokenToStringError; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum IngestError { + #[error("the generated token could not be detokenized: {0}")] + Detokenize(#[from] TokenToStringError), + #[error("the chat template's parser could not classify the answer: {0}")] + Classify(#[from] ParseChatMessageError), +} diff --git a/llama-cpp-bindings/src/error/json_object_failure.rs b/llama-cpp-bindings/src/error/json_object_failure.rs deleted file mode 100644 index c3bdc56f3..000000000 --- a/llama-cpp-bindings/src/error/json_object_failure.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum JsonObjectFailure { - #[error("tool call body has malformed JSON: {message}")] - InvalidJson { message: String }, -} diff --git a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs index 73c5373dc..ff1319fd4 100644 --- a/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs +++ b/llama-cpp-bindings/src/error/json_schema_to_grammar_error.rs @@ -1,16 +1,24 @@ use std::ffi::NulError; use std::string::FromUtf8Error; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum JsonSchemaToGrammarError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("schema string contains an interior NUL byte: {0}")] SchemaContainsNulByte(#[from] NulError), #[error("JSON schema is invalid: {message}")] InvalidSchema { message: String }, + #[error("JSON schema is not valid JSON: {message}")] + InvalidJson { message: String }, #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/key_value_xml_tags_failure.rs b/llama-cpp-bindings/src/error/key_value_xml_tags_failure.rs deleted file mode 100644 index 75960f526..000000000 --- a/llama-cpp-bindings/src/error/key_value_xml_tags_failure.rs +++ /dev/null @@ -1,30 +0,0 @@ -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum KeyValueXmlTagsFailure { - #[error("tool call function tag has empty name")] - EmptyFunctionName, - #[error("tool call function block is missing close tag '{expected_close}'")] - UnclosedFunctionBlock { expected_close: String }, - #[error("tool call function '{function_name}' has key tag with empty content")] - EmptyKey { function_name: String }, - #[error("tool call function '{function_name}' is missing key close tag '{expected_close}'")] - UnclosedKeyTag { - function_name: String, - expected_close: String, - }, - #[error( - "tool call function '{function_name}' key '{key}' is missing value open tag '{expected_open}'" - )] - MissingValueTag { - function_name: String, - key: String, - expected_open: String, - }, - #[error( - "tool call function '{function_name}' key '{key}' is missing value close tag '{expected_close}'" - )] - UnclosedValueTag { - function_name: String, - key: String, - expected_close: String, - }, -} diff --git a/llama-cpp-bindings/src/error/kv_cache_seq_pos_max_error.rs b/llama-cpp-bindings/src/error/kv_cache_seq_pos_max_error.rs deleted file mode 100644 index 0c8fbba6a..000000000 --- a/llama-cpp-bindings/src/error/kv_cache_seq_pos_max_error.rs +++ /dev/null @@ -1,17 +0,0 @@ -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum KvCacheSeqPosMaxError { - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), - #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error("context has no memory module available")] - MemoryHandleUnavailable, - #[error("sequence id {seq_id} is outside the context sequence range")] - SequenceIdOutOfRange { seq_id: i32 }, - #[error("not enough memory")] - NotEnoughMemory, - #[error("the llama.cpp library ran out of memory")] - LlamaCppOutOfMemory, - #[error("{message}")] - Reported { message: String }, -} diff --git a/llama-cpp-bindings/src/error/llama_backend_device_error.rs b/llama-cpp-bindings/src/error/llama_backend_device_error.rs new file mode 100644 index 000000000..25f9c550f --- /dev/null +++ b/llama-cpp-bindings/src/error/llama_backend_device_error.rs @@ -0,0 +1,14 @@ +use crate::error::backend_device_text_error::BackendDeviceTextError; +use crate::error::unknown_backend_device_type::UnknownBackendDeviceType; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum LlamaBackendDeviceError { + #[error("the device name cannot be read: {0}")] + Name(#[source] BackendDeviceTextError), + #[error("the device description cannot be read: {0}")] + Description(#[source] BackendDeviceTextError), + #[error("the device's backend name cannot be read: {0}")] + BackendName(#[source] BackendDeviceTextError), + #[error(transparent)] + UnknownDeviceType(#[from] UnknownBackendDeviceType), +} diff --git a/llama-cpp-bindings/src/error/llama_context_load_error.rs b/llama-cpp-bindings/src/error/llama_context_load_error.rs index b0145ce37..efd6c7671 100644 --- a/llama-cpp-bindings/src/error/llama_context_load_error.rs +++ b/llama-cpp-bindings/src/error/llama_context_load_error.rs @@ -1,9 +1,18 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::error::unknown_attention_type::UnknownAttentionType; +use crate::error::unknown_pooling_type::UnknownPoolingType; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LlamaContextLoadError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), #[error("context could not be constructed")] Unconstructible, #[error("not enough memory")] @@ -19,4 +28,8 @@ pub enum LlamaContextLoadError { n_seq_max: u32, output_capacity: u32, }, + #[error(transparent)] + UnknownAttentionType(#[from] UnknownAttentionType), + #[error(transparent)] + UnknownPoolingType(#[from] UnknownPoolingType), } diff --git a/llama-cpp-bindings/src/error/llama_cpp_error.rs b/llama-cpp-bindings/src/error/llama_cpp_error.rs index e40664a8a..12341da21 100644 --- a/llama-cpp-bindings/src/error/llama_cpp_error.rs +++ b/llama-cpp-bindings/src/error/llama_cpp_error.rs @@ -8,7 +8,7 @@ use crate::error::json_schema_to_grammar_error::JsonSchemaToGrammarError; use crate::error::llama_context_load_error::LlamaContextLoadError; use crate::error::llama_model_load_error::LlamaModelLoadError; -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LlamaCppError { #[error("BackendAlreadyInitialized")] BackendAlreadyInitialized, diff --git a/llama-cpp-bindings/src/error/llama_model_load_error.rs b/llama-cpp-bindings/src/error/llama_model_load_error.rs index 38b676d93..340e8fcab 100644 --- a/llama-cpp-bindings/src/error/llama_model_load_error.rs +++ b/llama-cpp-bindings/src/error/llama_model_load_error.rs @@ -1,12 +1,18 @@ use std::ffi::NulError; use std::path::PathBuf; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum LlamaModelLoadError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("null byte in string {0}")] NullError(#[from] NulError), #[error("failed to convert path {0} to str")] diff --git a/llama-cpp-bindings/src/error/logits_error.rs b/llama-cpp-bindings/src/error/logits_error.rs index 8462a9b9e..f5ee5d815 100644 --- a/llama-cpp-bindings/src/error/logits_error.rs +++ b/llama-cpp-bindings/src/error/logits_error.rs @@ -1,13 +1,17 @@ +use crate::context::llama_pooling_type::LlamaPoolingType; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LogitsError { #[error("logits data pointer is null")] NullLogits, + #[error( + "the last decode pooled embeddings with {pooling_type:?} pooling, which produces no logits" + )] + PooledEmbeddingsProduceNoLogits { pooling_type: LlamaPoolingType }, + #[error("the model graph of the last decode produces no logits")] + GraphProducesNoLogits, #[error("logit for token index {0} is not initialized")] TokenNotInitialized(i32), - #[error("token index {token_index} exceeds context size {context_size}")] - TokenIndexExceedsContext { token_index: u32, context_size: u32 }, #[error("n_vocab does not fit into usize: {0}")] VocabSizeOverflow(#[source] std::num::TryFromIntError), - #[error("token_index does not fit into u32: {0}")] - TokenIndexOverflow(#[source] std::num::TryFromIntError), } diff --git a/llama-cpp-bindings/src/error/marker_detection_error.rs b/llama-cpp-bindings/src/error/marker_detection_error.rs deleted file mode 100644 index c3740617e..000000000 --- a/llama-cpp-bindings/src/error/marker_detection_error.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::str::Utf8Error; -use std::string::FromUtf8Error; - -use crate::error::chat_template_error::ChatTemplateError; -use crate::error::string_to_token_error::StringToTokenError; -use crate::token::LlamaToken; - -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum MarkerDetectionError { - #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), - #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), - #[error("ffi returned non-utf8 marker bytes: {0}")] - MarkerUtf8Error(#[from] FromUtf8Error), - #[error("not enough memory")] - NotEnoughMemory, - #[error("{operation} could not run because the model has no chat template")] - ModelHasNoChatTemplate { operation: &'static str }, - #[error("{operation} could not run because the model has no vocab")] - ModelHasNoVocab { operation: &'static str }, - #[error("the llama.cpp library ran out of memory")] - LlamaCppOutOfMemory, - #[error("reasoning-marker detection failed: {message}")] - ReasoningMarkerDetectionFailed { message: String }, - #[error("tool-call haystack computation failed: {message}")] - ToolCallHaystackComputationFailed { message: String }, - #[error("tool-call synthetic-render diagnosis failed: {message}")] - ToolCallSyntheticRenderDiagnosisFailed { message: String }, - #[error("the reasoning-markers destructor threw: {message}")] - ReasoningMarkersFreeFailed { message: String }, - #[error("a detected marker string could not be tokenised: {0}")] - MarkerTokenizationFailed(#[from] StringToTokenError), - #[error("marker detection produced an empty token sequence")] - EmptyMarker, - #[error("marker token sequence {tokens:?} opens more than one section")] - AmbiguousMarkerOpeners { tokens: Vec }, - #[error("the chat template is not valid UTF-8: {0}")] - ToolCallTemplateNotUtf8(#[from] Utf8Error), - #[error("the chat template could not be retrieved for tool-call marker detection: {0}")] - ChatTemplateUnavailable(#[source] ChatTemplateError), - #[error("{operation} rejected the Rust-owned argument {argument}")] - WrapperRejectedArgument { - operation: &'static str, - argument: &'static str, - }, -} diff --git a/llama-cpp-bindings/src/error/kv_cache_seq_div_error.rs b/llama-cpp-bindings/src/error/memory_seq_add_error.rs similarity index 53% rename from llama-cpp-bindings/src/error/kv_cache_seq_div_error.rs rename to llama-cpp-bindings/src/error/memory_seq_add_error.rs index 463d345f0..1040a5cd3 100644 --- a/llama-cpp-bindings/src/error/kv_cache_seq_div_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_add_error.rs @@ -1,11 +1,20 @@ use std::num::TryFromIntError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::unknown_rope_type::UnknownRopeType; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum KvCacheSeqDivError { +pub enum MemorySeqAddError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("provided start position is too large for an i32")] P0TooLarge(#[source] TryFromIntError), #[error("provided end position is too large for an i32")] @@ -14,6 +23,10 @@ pub enum KvCacheSeqDivError { IncompatibleRopeType, #[error("context has no memory module available")] MemoryHandleUnavailable, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), + #[error(transparent)] + UnknownRopeType(#[from] UnknownRopeType), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/copy_kv_cache_seq_error.rs b/llama-cpp-bindings/src/error/memory_seq_cp_error.rs similarity index 68% rename from llama-cpp-bindings/src/error/copy_kv_cache_seq_error.rs rename to llama-cpp-bindings/src/error/memory_seq_cp_error.rs index 011e70744..863ffa4e4 100644 --- a/llama-cpp-bindings/src/error/copy_kv_cache_seq_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_cp_error.rs @@ -1,11 +1,15 @@ use std::num::TryFromIntError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum CopyKvCacheSeqError { +pub enum MemorySeqCpError { #[error("provided start position is too large for an i32")] P0TooLarge(#[source] TryFromIntError), #[error("provided end position is too large for an i32")] P1TooLarge(#[source] TryFromIntError), #[error("the context has no memory module attached")] MemoryHandleUnavailable, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), } diff --git a/llama-cpp-bindings/src/error/kv_cache_seq_add_error.rs b/llama-cpp-bindings/src/error/memory_seq_div_error.rs similarity index 53% rename from llama-cpp-bindings/src/error/kv_cache_seq_add_error.rs rename to llama-cpp-bindings/src/error/memory_seq_div_error.rs index a018a705f..5aa855023 100644 --- a/llama-cpp-bindings/src/error/kv_cache_seq_add_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_div_error.rs @@ -1,11 +1,20 @@ use std::num::TryFromIntError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; +use crate::error::unknown_rope_type::UnknownRopeType; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum KvCacheSeqAddError { +pub enum MemorySeqDivError { + #[error(transparent)] + CppString(#[from] CppStringError), #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("provided start position is too large for an i32")] P0TooLarge(#[source] TryFromIntError), #[error("provided end position is too large for an i32")] @@ -14,6 +23,10 @@ pub enum KvCacheSeqAddError { IncompatibleRopeType, #[error("context has no memory module available")] MemoryHandleUnavailable, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), + #[error(transparent)] + UnknownRopeType(#[from] UnknownRopeType), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs new file mode 100644 index 000000000..44f040c04 --- /dev/null +++ b/llama-cpp-bindings/src/error/memory_seq_pos_max_error.rs @@ -0,0 +1,25 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum MemorySeqPosMaxError { + #[error(transparent)] + CppString(#[from] CppStringError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), + #[error("context has no memory module available")] + MemoryHandleUnavailable, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/error/clear_kv_cache_seq_error.rs b/llama-cpp-bindings/src/error/memory_seq_rm_error.rs similarity index 76% rename from llama-cpp-bindings/src/error/clear_kv_cache_seq_error.rs rename to llama-cpp-bindings/src/error/memory_seq_rm_error.rs index 45c01bcdc..22560d2e7 100644 --- a/llama-cpp-bindings/src/error/clear_kv_cache_seq_error.rs +++ b/llama-cpp-bindings/src/error/memory_seq_rm_error.rs @@ -1,8 +1,10 @@ use std::ffi::c_int; use std::num::TryFromIntError; +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum ClearKvCacheSeqError { +pub enum MemorySeqRmError { #[error("provided sequence id is too large for an i32")] SeqIdTooLarge(#[source] TryFromIntError), #[error("provided start position is too large for an i32")] @@ -11,4 +13,6 @@ pub enum ClearKvCacheSeqError { P1TooLarge(#[source] TryFromIntError), #[error("sequence {seq_id} could not be partially removed over positions [{p0}, {p1})")] PartialSequenceNotRemoved { seq_id: c_int, p0: c_int, p1: c_int }, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), } diff --git a/llama-cpp-bindings/src/error/model_params_error.rs b/llama-cpp-bindings/src/error/model_params_error.rs index 8e70ebb4f..a8ad8ee1b 100644 --- a/llama-cpp-bindings/src/error/model_params_error.rs +++ b/llama-cpp-bindings/src/error/model_params_error.rs @@ -4,6 +4,9 @@ pub enum ModelParamsError { NoAvailableSlot, #[error("Override slot is not empty")] SlotNotEmpty, - #[error("Invalid character in key: byte {byte}, {reason}")] - InvalidCharacterInKey { byte: u8, reason: String }, + #[error("the key needs {length_with_nul} bytes with its NUL, but llama.cpp holds {capacity}")] + KeyTooLong { + length_with_nul: usize, + capacity: usize, + }, } diff --git a/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs b/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs index d38337bfe..008e7c434 100644 --- a/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs +++ b/llama-cpp-bindings/src/error/new_llama_chat_message_error.rs @@ -1,6 +1,6 @@ use std::ffi::NulError; -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum NewLlamaChatMessageError { #[error("{0}")] NulError(#[from] NulError), diff --git a/llama-cpp-bindings/src/error/paired_quote_failure.rs b/llama-cpp-bindings/src/error/paired_quote_failure.rs deleted file mode 100644 index a1d8fc510..000000000 --- a/llama-cpp-bindings/src/error/paired_quote_failure.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum PairedQuoteFailure { - #[error("empty key in tool call '{tool_name}' arguments")] - EmptyKey { tool_name: String }, - #[error("tool call '{tool_name}' translated arguments are not valid JSON: {message}")] - InvalidJsonArguments { tool_name: String, message: String }, - #[error("tool call '{tool_name}' has unclosed quoted value for key '{key}'")] - UnclosedQuotedValue { tool_name: String, key: String }, - #[error("tool call '{tool_name}' arguments ended without close marker (state: {state})")] - UnclosedArgumentBlock { - tool_name: String, - state: &'static str, - }, - #[error( - "tool call '{tool_name}' has unexpected character '{character}' after value for key '{key}'" - )] - UnexpectedCharAfterValue { - tool_name: String, - key: String, - character: char, - }, -} diff --git a/llama-cpp-bindings/src/error/parse_chat_message_error.rs b/llama-cpp-bindings/src/error/parse_chat_message_error.rs index 3287d137d..7ecfd4852 100644 --- a/llama-cpp-bindings/src/error/parse_chat_message_error.rs +++ b/llama-cpp-bindings/src/error/parse_chat_message_error.rs @@ -1,45 +1,23 @@ -use std::string::FromUtf8Error; +use std::str::Utf8Error; -use crate::error::marker_detection_error::MarkerDetectionError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum ParseChatMessageError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), - #[error("model has no chat template")] - NoChatTemplate, - #[error("model has no vocab")] - NoVocab, + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] LlamaCppOutOfMemory, - #[error("the chat parser could not be constructed: {message}")] - ParserCreationFailed { message: String }, - #[error("the chat tools contain an interior NUL byte")] - ToolsContainNulByte(#[source] std::ffi::NulError), - #[error("the chat tools are not a JSON array")] - ToolsNotAnArray, - #[error("the chat parser could not be built for the given tools: {message}")] - ToolsParserBuildFailed { message: String }, - #[error("the chat parser did not recognize the message: {message}")] - MessageUnrecognized { message: String }, - #[error("the chat parser destructor threw: {message}")] - DestructorFailed { message: String }, - #[error("tool-call id index {index} out of bounds")] - ToolCallIdIndexOutOfBounds { index: usize }, - #[error("tool-call name index {index} out of bounds")] - ToolCallNameIndexOutOfBounds { index: usize }, - #[error("tool-call arguments index {index} out of bounds")] - ToolCallArgumentsIndexOutOfBounds { index: usize }, - #[error("ffi returned non-utf8 string: {0}")] - StringUtf8Error(#[from] FromUtf8Error), - #[error("the message to parse contains an interior NUL byte")] - InputContainsNulByte(#[source] std::ffi::NulError), - #[error("reasoning-marker detection failed: {0}")] - MarkerDetection(#[from] MarkerDetectionError), + #[error("text llama.cpp parsed out of the answer is not valid UTF-8: {0}")] + ParsedTextNotUtf8(#[source] Utf8Error), #[error("{message}")] Reported { message: String }, } diff --git a/llama-cpp-bindings/src/error/process_failure.rs b/llama-cpp-bindings/src/error/process_failure.rs new file mode 100644 index 000000000..c5f2663cd --- /dev/null +++ b/llama-cpp-bindings/src/error/process_failure.rs @@ -0,0 +1,20 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +/// How `llama_rs_process` failed before llama.cpp returned a code of its own. +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +pub enum ProcessFailure { + #[error(transparent)] + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/error/sample_error.rs b/llama-cpp-bindings/src/error/sample_error.rs index 37d28de86..7b1a1960c 100644 --- a/llama-cpp-bindings/src/error/sample_error.rs +++ b/llama-cpp-bindings/src/error/sample_error.rs @@ -1,20 +1,29 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::error::ingest_error::IngestError; +use crate::error::logits_error::LogitsError; use crate::error::sampler_apply_error::SamplerApplyError; -use crate::error::token_to_string_error::TokenToStringError; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum SampleError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), + #[error("the logits to sample from are unavailable: {0}")] + Logits(#[from] LogitsError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] LlamaCppOutOfMemory, #[error("applying the sampler to the token data array failed: {0}")] SamplerApply(#[from] SamplerApplyError), - #[error("token detokenization failed during classification: {0}")] - Detokenize(#[from] TokenToStringError), + #[error("classifying the sampled token failed: {0}")] + Ingest(#[from] IngestError), #[error("the grammar sampler callback failed during sampling: {message}")] GrammarCallbackFailed { message: String }, #[error("{message}")] diff --git a/llama-cpp-bindings/src/error/sampler_accept_error.rs b/llama-cpp-bindings/src/error/sampler_accept_error.rs index b8865984a..cb7e590b4 100644 --- a/llama-cpp-bindings/src/error/sampler_accept_error.rs +++ b/llama-cpp-bindings/src/error/sampler_accept_error.rs @@ -1,9 +1,15 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum SamplerAcceptError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-bindings/src/error/sampler_apply_error.rs b/llama-cpp-bindings/src/error/sampler_apply_error.rs index e7d5c1428..daddb2e61 100644 --- a/llama-cpp-bindings/src/error/sampler_apply_error.rs +++ b/llama-cpp-bindings/src/error/sampler_apply_error.rs @@ -1,9 +1,15 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] pub enum SamplerApplyError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), #[error("the sampler pointer was null when applying to the token data array")] NullSampler, #[error("the sampler excluded every candidate, leaving no token to select")] diff --git a/llama-cpp-bindings/src/error/sampling_error.rs b/llama-cpp-bindings/src/error/sampling_error.rs index 8f7fb8ac5..9c5631fa4 100644 --- a/llama-cpp-bindings/src/error/sampling_error.rs +++ b/llama-cpp-bindings/src/error/sampling_error.rs @@ -1,9 +1,17 @@ +use std::ffi::NulError; + +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum SamplingError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), #[error("a value does not fit into i32")] IntegerOverflow(#[source] std::num::TryFromIntError), #[error("the llama.cpp library ran out of memory initializing the {sampler} sampler")] @@ -17,4 +25,24 @@ pub enum SamplingError { }, #[error("{sampler} sampler could not be initialized")] SamplerUnavailable { sampler: &'static str }, + #[error("DRY sequence breaker {index} is empty")] + DrySequenceBreakerEmpty { index: usize }, + #[error( + "DRY sequence breaker {index} is {length} bytes long, more than the {max_length} bytes llama.cpp uses" + )] + DrySequenceBreakerTooLong { + index: usize, + length: usize, + max_length: usize, + }, + #[error("DRY sequence breaker {index} contains an interior NUL byte")] + DrySequenceBreakerContainsNul { + index: usize, + #[source] + source: NulError, + }, + #[error( + "{distinct} distinct DRY sequence breakers were given, more than the {max} llama.cpp uses" + )] + TooManyDistinctDrySequenceBreakers { distinct: usize, max: usize }, } diff --git a/llama-cpp-bindings/src/error/sequence_id_out_of_range.rs b/llama-cpp-bindings/src/error/sequence_id_out_of_range.rs new file mode 100644 index 000000000..f23f40676 --- /dev/null +++ b/llama-cpp-bindings/src/error/sequence_id_out_of_range.rs @@ -0,0 +1,6 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("sequence id {seq_id} names no sequence of a context with {n_seq_max} sequences")] +pub struct SequenceIdOutOfRange { + pub seq_id: i32, + pub n_seq_max: u32, +} diff --git a/llama-cpp-bindings/src/error/string_to_token_error.rs b/llama-cpp-bindings/src/error/string_to_token_error.rs index 8d9e5dd08..082f33dd2 100644 --- a/llama-cpp-bindings/src/error/string_to_token_error.rs +++ b/llama-cpp-bindings/src/error/string_to_token_error.rs @@ -1,11 +1,17 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum StringToTokenError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("{0}")] NulError(#[from] NulError), #[error("{0}")] diff --git a/llama-cpp-bindings/src/error/token_outside_vocabulary.rs b/llama-cpp-bindings/src/error/token_outside_vocabulary.rs index 6598c8b06..553c7d86e 100644 --- a/llama-cpp-bindings/src/error/token_outside_vocabulary.rs +++ b/llama-cpp-bindings/src/error/token_outside_vocabulary.rs @@ -6,3 +6,17 @@ pub struct TokenOutsideVocabulary { pub token: LlamaToken, pub n_vocab: i32, } + +impl TokenOutsideVocabulary { + /// # Errors + /// + /// Returns [`TokenOutsideVocabulary`] when `token` is not one of the `n_vocab` tokens of the + /// vocabulary. + pub fn check(token: LlamaToken, n_vocab: i32) -> Result { + if (0..n_vocab).contains(&token.0) { + Ok(token) + } else { + Err(Self { token, n_vocab }) + } + } +} diff --git a/llama-cpp-bindings/src/error/token_sampling_error.rs b/llama-cpp-bindings/src/error/token_sampling_error.rs deleted file mode 100644 index 993507620..000000000 --- a/llama-cpp-bindings/src/error/token_sampling_error.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::SamplingError; -use crate::error::sampler_apply_error::SamplerApplyError; - -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum TokenSamplingError { - #[error("sampler initialization failed: {0}")] - SamplerInitialization(#[from] SamplingError), - #[error("No token was selected by the sampler")] - NoTokenSelected, - #[error("applying the sampler to the token data array failed: {0}")] - SamplerApply(#[from] SamplerApplyError), -} diff --git a/llama-cpp-bindings/src/error/token_to_string_error.rs b/llama-cpp-bindings/src/error/token_to_string_error.rs index 85c12645d..6e572ce5e 100644 --- a/llama-cpp-bindings/src/error/token_to_string_error.rs +++ b/llama-cpp-bindings/src/error/token_to_string_error.rs @@ -1,4 +1,4 @@ -use std::os::raw::c_int; +use std::ffi::c_int; use std::string::FromUtf8Error; use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; @@ -8,8 +8,6 @@ use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; pub enum TokenToStringError { #[error(transparent)] TokenOutsideVocabulary(#[from] TokenOutsideVocabulary), - #[error("Unknown Token Type")] - UnknownTokenType, #[error("Insufficient Buffer Space {0}")] InsufficientBufferSpace(c_int), #[error("FromUtf8Error {0}")] diff --git a/llama-cpp-bindings/src/error/tool_call_format_failure.rs b/llama-cpp-bindings/src/error/tool_call_format_failure.rs deleted file mode 100644 index 71bbcfbe3..000000000 --- a/llama-cpp-bindings/src/error/tool_call_format_failure.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::error::bracketed_args_failure::BracketedArgsFailure; -use crate::error::json_object_failure::JsonObjectFailure; -use crate::error::key_value_xml_tags_failure::KeyValueXmlTagsFailure; -use crate::error::paired_quote_failure::PairedQuoteFailure; -use crate::error::xml_function_tags_failure::XmlFunctionTagsFailure; - -#[derive(Debug, Eq, PartialEq, thiserror::Error)] -pub enum ToolCallFormatFailure { - #[error("bracketed-args parser: {0}")] - BracketedArgs(#[from] BracketedArgsFailure), - #[error("json-object parser: {0}")] - JsonObject(#[from] JsonObjectFailure), - #[error("key-value-xml-tags parser: {0}")] - KeyValueXmlTags(#[from] KeyValueXmlTagsFailure), - #[error("paired-quote parser: {0}")] - PairedQuote(#[from] PairedQuoteFailure), - #[error("xml-function-tags parser: {0}")] - XmlFunctionTags(#[from] XmlFunctionTagsFailure), -} diff --git a/llama-cpp-bindings/src/error/unknown_attention_type.rs b/llama-cpp-bindings/src/error/unknown_attention_type.rs new file mode 100644 index 000000000..d48f1858d --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_attention_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("attention type {0} is not a value llama.cpp defines")] +pub struct UnknownAttentionType(pub llama_cpp_bindings_sys::llama_attention_type); diff --git a/llama-cpp-bindings/src/error/unknown_backend_device_type.rs b/llama-cpp-bindings/src/error/unknown_backend_device_type.rs new file mode 100644 index 000000000..35fcca137 --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_backend_device_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("backend device type {0} is not a value ggml defines")] +pub struct UnknownBackendDeviceType(pub llama_cpp_bindings_sys::ggml_backend_dev_type); diff --git a/llama-cpp-bindings/src/error/unknown_pooling_type.rs b/llama-cpp-bindings/src/error/unknown_pooling_type.rs new file mode 100644 index 000000000..69c18752c --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_pooling_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("pooling type {0} is not a value llama.cpp defines")] +pub struct UnknownPoolingType(pub llama_cpp_bindings_sys::llama_pooling_type); diff --git a/llama-cpp-bindings/src/error/unknown_rope_scaling_type.rs b/llama-cpp-bindings/src/error/unknown_rope_scaling_type.rs new file mode 100644 index 000000000..fb523f16e --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_rope_scaling_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("rope scaling type {0} is not a value llama.cpp defines")] +pub struct UnknownRopeScalingType(pub llama_cpp_bindings_sys::llama_rope_scaling_type); diff --git a/llama-cpp-bindings/src/error/unknown_rope_type.rs b/llama-cpp-bindings/src/error/unknown_rope_type.rs new file mode 100644 index 000000000..b09934d44 --- /dev/null +++ b/llama-cpp-bindings/src/error/unknown_rope_type.rs @@ -0,0 +1,3 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("rope type {0} is not a value llama.cpp defines")] +pub struct UnknownRopeType(pub llama_cpp_bindings_sys::llama_rope_type); diff --git a/llama-cpp-bindings/src/error/xml_function_tags_failure.rs b/llama-cpp-bindings/src/error/xml_function_tags_failure.rs deleted file mode 100644 index aa8314a7b..000000000 --- a/llama-cpp-bindings/src/error/xml_function_tags_failure.rs +++ /dev/null @@ -1,20 +0,0 @@ -#[derive(Debug, PartialEq, Eq, thiserror::Error)] -pub enum XmlFunctionTagsFailure { - #[error("tool call function tag has empty name")] - EmptyFunctionName, - #[error("tool call function '{function_name}' is missing close tag '{expected_close}'")] - UnclosedFunctionBlock { - function_name: String, - expected_close: String, - }, - #[error("tool call function '{function_name}' has parameter with empty name")] - EmptyParameterName { function_name: String }, - #[error( - "tool call function '{function_name}' parameter '{parameter_name}' is missing close tag '{expected_close}'" - )] - UnclosedParameterBlock { - function_name: String, - parameter_name: String, - expected_close: String, - }, -} diff --git a/llama-cpp-bindings/src/extract_tool_call_markers_from_haystack.rs b/llama-cpp-bindings/src/extract_tool_call_markers_from_haystack.rs deleted file mode 100644 index bbbbdb4e6..000000000 --- a/llama-cpp-bindings/src/extract_tool_call_markers_from_haystack.rs +++ /dev/null @@ -1,143 +0,0 @@ -use crate::tool_call_marker_pair::ToolCallMarkerPair; - -#[must_use] -pub fn extract_tool_call_markers_from_haystack(haystack: &str) -> Option { - if haystack.is_empty() { - return None; - } - - let json_start = haystack.find('{')?; - let json_end = haystack.rfind('}')?; - if json_end < json_start { - return None; - } - - let json_slice = &haystack[json_start..=json_end]; - serde_json::from_str::(json_slice).ok()?; - - let open = haystack[..json_start].trim().to_owned(); - let close = haystack[json_end + 1..].trim().to_owned(); - - if open.is_empty() || close.is_empty() { - return None; - } - - Some(ToolCallMarkerPair { open, close }) -} - -#[cfg(test)] -mod tests { - use super::ToolCallMarkerPair; - use super::extract_tool_call_markers_from_haystack; - - #[test] - fn extracts_open_and_close_around_a_simple_json_payload() { - let pair = extract_tool_call_markers_from_haystack( - "{\"name\":\"x\",\"arguments\":{}}", - ); - - assert_eq!( - pair, - Some(ToolCallMarkerPair { - open: "".to_owned(), - close: "".to_owned(), - }), - ); - } - - #[test] - fn trims_surrounding_whitespace_from_each_marker() { - let pair = extract_tool_call_markers_from_haystack( - " \n {\"k\": 1}\n ", - ); - - assert_eq!( - pair, - Some(ToolCallMarkerPair { - open: "".to_owned(), - close: "".to_owned(), - }), - ); - } - - #[test] - fn returns_none_when_haystack_is_empty() { - assert_eq!(extract_tool_call_markers_from_haystack(""), None); - } - - #[test] - fn returns_none_when_haystack_has_no_open_brace() { - assert_eq!( - extract_tool_call_markers_from_haystack("plain assistant text"), - None - ); - } - - #[test] - fn returns_none_when_haystack_has_open_brace_but_no_close() { - assert_eq!( - extract_tool_call_markers_from_haystack("{ unclosed"), - None - ); - } - - #[test] - fn returns_none_when_close_brace_precedes_open_brace() { - assert_eq!( - extract_tool_call_markers_from_haystack("}{"), - None - ); - } - - #[test] - fn returns_none_when_brace_payload_is_not_valid_json() { - assert_eq!( - extract_tool_call_markers_from_haystack("{not valid json}"), - None - ); - } - - #[test] - fn returns_none_when_open_marker_resolves_to_empty_after_trim() { - assert_eq!( - extract_tool_call_markers_from_haystack(" {\"x\":1}"), - None - ); - } - - #[test] - fn returns_none_when_close_marker_resolves_to_empty_after_trim() { - assert_eq!( - extract_tool_call_markers_from_haystack("{\"x\":1} "), - None - ); - } - - #[test] - fn extracts_around_an_object_that_contains_nested_braces() { - let pair = extract_tool_call_markers_from_haystack( - "{\"args\":{\"k\":[1,2,{\"deep\":true}]}}", - ); - - assert_eq!( - pair, - Some(ToolCallMarkerPair { - open: "".to_owned(), - close: "".to_owned(), - }), - ); - } - - #[test] - fn extracts_when_open_marker_contains_multibyte_utf8() { - let pair = extract_tool_call_markers_from_haystack("<|tool→call|>{\"k\":1}<|/tool→call|>"); - - assert_eq!( - pair, - Some(ToolCallMarkerPair { - open: "<|tool→call|>".to_owned(), - close: "<|/tool→call|>".to_owned(), - }), - ); - } -} diff --git a/llama-cpp-bindings/src/ggml_time_us.rs b/llama-cpp-bindings/src/ggml_time_us.rs deleted file mode 100644 index 4d9db374d..000000000 --- a/llama-cpp-bindings/src/ggml_time_us.rs +++ /dev/null @@ -1,21 +0,0 @@ -#[must_use] -pub fn ggml_time_us() -> i64 { - unsafe { llama_cpp_bindings_sys::ggml_time_us() } -} - -#[cfg(test)] -mod tests { - use serial_test::serial; - - use super::ggml_time_us; - use crate::llama_backend::LlamaBackend; - - #[test] - #[serial] - fn returns_positive_value() { - let _backend = LlamaBackend::init().unwrap(); - let time_microseconds = ggml_time_us(); - - assert!(time_microseconds > 0); - } -} diff --git a/llama-cpp-bindings/src/gguf_context.rs b/llama-cpp-bindings/src/gguf_context.rs index 32ec8cd72..765060594 100644 --- a/llama-cpp-bindings/src/gguf_context.rs +++ b/llama-cpp-bindings/src/gguf_context.rs @@ -4,6 +4,7 @@ use std::fs::File; use std::io; use std::os::unix::fs::FileExt as _; use std::path::{Path, PathBuf}; +use std::ptr; use std::ptr::NonNull; use std::slice; @@ -34,7 +35,7 @@ impl GgufContext { let init_params = llama_cpp_bindings_sys::gguf_init_params { no_alloc: true, - ctx: std::ptr::null_mut(), + ctx: ptr::null_mut(), }; let raw = @@ -400,9 +401,20 @@ impl Drop for GgufContext { #[cfg(test)] mod tests { + use std::env; use std::ffi::CString; + use std::ffi::OsStr; + use std::fs; + use std::fs::File; + use std::io; + use std::io::Write as _; use std::mem::Discriminant; + use std::mem::discriminant; + use std::os::unix::ffi::OsStrExt; + use std::path::Path; use std::path::PathBuf; + use std::process; + use std::str; use super::GgufContext; use crate::gguf_context_error::GgufContextError; @@ -418,28 +430,28 @@ mod tests { } fn init_failed_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::InitFailed(PathBuf::new())) + discriminant(&GgufContextError::InitFailed(PathBuf::new())) } fn key_not_found_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::KeyNotFound { key: String::new() }) + discriminant(&GgufContextError::KeyNotFound { key: String::new() }) } fn nul_error_disc() -> Discriminant { let nul_err = CString::new(b"a\0b".to_vec()).unwrap_err(); - std::mem::discriminant(&GgufContextError::NulError(nul_err)) + discriminant(&GgufContextError::NulError(nul_err)) } fn path_to_str_error_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::PathToStrError(PathBuf::new())) + discriminant(&GgufContextError::PathToStrError(PathBuf::new())) } fn key_id_out_of_range_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::KeyIdOutOfRange { key_id: 0, n_kv: 0 }) + discriminant(&GgufContextError::KeyIdOutOfRange { key_id: 0, n_kv: 0 }) } fn value_type_mismatch_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::ValueTypeMismatch { + discriminant(&GgufContextError::ValueTypeMismatch { key_id: 0, expected: GgufType::Uint32, actual: GgufType::Uint32, @@ -447,20 +459,20 @@ mod tests { } fn tensor_not_found_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::TensorNotFound { + discriminant(&GgufContextError::TensorNotFound { name: String::new(), }) } fn tensor_is_not_f32_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::TensorIsNotF32 { + discriminant(&GgufContextError::TensorIsNotF32 { name: String::new(), ggml_type: 0, }) } fn array_element_type_mismatch_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::ArrayElementTypeMismatch { + discriminant(&GgufContextError::ArrayElementTypeMismatch { key_id: 0, expected: GgufType::Int32, actual_raw_type: 0, @@ -480,14 +492,14 @@ mod tests { } fn source_file_read_failed_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::SourceFileReadFailed { + discriminant(&GgufContextError::SourceFileReadFailed { path: PathBuf::new(), - source: std::io::Error::other("discriminant"), + source: io::Error::other("discriminant"), }) } fn source_file_truncated_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::SourceFileTruncated { + discriminant(&GgufContextError::SourceFileTruncated { path: PathBuf::new(), data_offset: 0, file_size: 0, @@ -495,32 +507,32 @@ mod tests { } fn destination_file_write_failed_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::DestinationFileWriteFailed { + discriminant(&GgufContextError::DestinationFileWriteFailed { path: PathBuf::new(), - source: std::io::Error::other("discriminant"), + source: io::Error::other("discriminant"), }) } fn rewritten_gguf_path(test_name: &str) -> PathBuf { - std::env::temp_dir().join(format!( + env::temp_dir().join(format!( "llama_cpp_bindings_rewritten_{}_{}.gguf", - std::process::id(), + process::id(), test_name, )) } fn tensor_read_failed_disc() -> Discriminant { - std::mem::discriminant(&GgufContextError::TensorReadFailed { + discriminant(&GgufContextError::TensorReadFailed { name: String::new(), path: PathBuf::new(), - source: std::io::Error::other("discriminant"), + source: io::Error::other("discriminant"), }) } fn utf8_error_disc() -> Discriminant { let invalid_utf8_bytes: Vec = vec![0xFF]; - let utf8_err = std::str::from_utf8(&invalid_utf8_bytes).unwrap_err(); - std::mem::discriminant(&GgufContextError::Utf8Error(utf8_err)) + let utf8_err = str::from_utf8(&invalid_utf8_bytes).unwrap_err(); + discriminant(&GgufContextError::Utf8Error(utf8_err)) } struct SyntheticTensor { @@ -603,15 +615,13 @@ mod tests { impl SyntheticGgufFile { fn from_bytes(test_name: &str, bytes: &[u8]) -> Self { - use std::io::Write as _; - - let path = std::env::temp_dir().join(format!( + let path = env::temp_dir().join(format!( "llama_cpp_bindings_synthetic_{}_{}.gguf", - std::process::id(), + process::id(), test_name, )); - let mut file = std::fs::File::create(&path).unwrap(); + let mut file = File::create(&path).unwrap(); file.write_all(bytes).unwrap(); Self { path } @@ -620,8 +630,7 @@ mod tests { impl Drop for SyntheticGgufFile { fn drop(&mut self) { - std::fs::remove_file(&self.path) - .unwrap_or_else(|error| panic!("failed to remove synthetic GGUF: {error}")); + fs::remove_file(&self.path).unwrap(); } } @@ -632,34 +641,30 @@ mod tests { .collect() } - #[test] - fn from_file_opens_valid_gguf() { - let context = GgufContext::from_file(fixture_path()); - - assert!(context.is_ok()); + fn bias_and_weight_tensors(test_name: &str) -> SyntheticGgufFile { + SyntheticGgufBuilder::default() + .tensor(b"synthetic.bias", &[2], GGML_TYPE_F16, vec![0; 4]) + .tensor( + b"synthetic.weight", + &[3, 2], + llama_cpp_bindings_sys::GGML_TYPE_F32, + f32_bytes(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + ) + .write(test_name) } #[test] fn from_file_nonexistent_returns_init_failed() { let err = GgufContext::from_file("/nonexistent/file.gguf").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), init_failed_disc()); + assert_eq!(discriminant(&err), init_failed_disc()); } #[test] - fn n_kv_returns_positive_count() { + fn n_kv_counts_every_key_of_the_file() { let context = GgufContext::from_file(fixture_path()).unwrap(); - assert!(context.n_kv() > 0); - } - - #[test] - fn find_key_returns_valid_index_for_known_key() { - let context = GgufContext::from_file(fixture_path()).unwrap(); - let index = context.find_key("general.architecture"); - - assert!(index.is_ok()); - assert!(index.unwrap() >= 0); + assert_eq!(context.n_kv(), 20); } #[test] @@ -667,7 +672,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.find_key("nonexistent.key").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), key_not_found_disc()); + assert_eq!(discriminant(&err), key_not_found_disc()); } #[test] @@ -684,7 +689,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.key_at(context.n_kv()).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), key_id_out_of_range_disc()); + assert_eq!(discriminant(&err), key_id_out_of_range_disc()); } #[test] @@ -692,7 +697,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.val_u32(-1).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), key_id_out_of_range_disc()); + assert_eq!(discriminant(&err), key_id_out_of_range_disc()); } #[test] @@ -704,30 +709,27 @@ mod tests { } #[test] - fn val_str_returns_architecture_value() { + fn val_str_reads_the_architecture_stored_in_the_file() { let context = GgufContext::from_file(fixture_path()).unwrap(); let index = context.find_key("general.architecture").unwrap(); let value = context.val_str(index).unwrap(); - assert_ne!(value, ""); + assert_eq!(value, "bert"); } #[test] fn from_file_non_utf8_path_returns_error() { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let non_utf8_path = std::path::Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); + let non_utf8_path = Path::new(OsStr::from_bytes(b"/tmp/\xff\xfe.gguf")); let err = GgufContext::from_file(non_utf8_path).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), path_to_str_error_disc()); + assert_eq!(discriminant(&err), path_to_str_error_disc()); } #[test] fn from_file_with_null_byte_in_path_returns_error() { let err = GgufContext::from_file("/tmp/foo\0bar.gguf").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), nul_error_disc()); + assert_eq!(discriminant(&err), nul_error_disc()); } #[test] @@ -735,11 +737,11 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.find_key("foo\0bar").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), nul_error_disc()); + assert_eq!(discriminant(&err), nul_error_disc()); } #[test] - fn typed_values_round_trip_through_synthetic_fixture() { + fn typed_getters_read_the_values_a_gguf_file_stores() { let fixture = SyntheticGgufBuilder::default() .value( b"synthetic.i32_value", @@ -791,10 +793,7 @@ mod tests { ]; for mismatch in mismatches { - assert_eq!( - std::mem::discriminant(&mismatch), - value_type_mismatch_disc() - ); + assert_eq!(discriminant(&mismatch), value_type_mismatch_disc()); } } @@ -808,7 +807,7 @@ mod tests { let value_index = context.find_key("synthetic.str_value").unwrap(); let err = context.val_str(value_index).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), utf8_error_disc()); + assert_eq!(discriminant(&err), utf8_error_disc()); } #[test] @@ -820,25 +819,24 @@ mod tests { let err = context.key_at(0).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), utf8_error_disc()); + assert_eq!(discriminant(&err), utf8_error_disc()); + } + + #[test] + fn n_tensors_counts_every_tensor_of_the_file() { + let fixture = bias_and_weight_tensors("n_tensors_counts_every_tensor_of_the_file"); + let context = GgufContext::from_file(&fixture.path).unwrap(); + + assert_eq!(context.n_tensors(), 2); } #[test] fn read_tensor_f32_returns_shape_and_values() { - let fixture = SyntheticGgufBuilder::default() - .tensor(b"synthetic.bias", &[2], GGML_TYPE_F16, vec![0; 4]) - .tensor( - b"synthetic.weight", - &[3, 2], - llama_cpp_bindings_sys::GGML_TYPE_F32, - f32_bytes(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), - ) - .write("read_tensor_f32_returns_shape_and_values"); + let fixture = bias_and_weight_tensors("read_tensor_f32_returns_shape_and_values"); let context = GgufContext::from_file(&fixture.path).unwrap(); let tensor = context.read_tensor_f32("synthetic.weight").unwrap(); - assert_eq!(context.n_tensors(), 2); assert_eq!(tensor.shape, [3, 2, 1, 1]); assert_eq!(tensor.values, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); } @@ -848,7 +846,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.read_tensor_f32("synthetic.missing").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), tensor_not_found_disc()); + assert_eq!(discriminant(&err), tensor_not_found_disc()); } #[test] @@ -859,7 +857,7 @@ mod tests { let context = GgufContext::from_file(&fixture.path).unwrap(); let err = context.read_tensor_f32("synthetic.half").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), tensor_is_not_f32_disc()); + assert_eq!(discriminant(&err), tensor_is_not_f32_disc()); } #[test] @@ -874,11 +872,11 @@ mod tests { .write("read_tensor_f32_reports_a_file_truncated_after_opening"); let context = GgufContext::from_file(&fixture.path).unwrap(); - std::fs::write(&fixture.path, b"").unwrap(); + fs::write(&fixture.path, b"").unwrap(); let err = context.read_tensor_f32("synthetic.weight").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), tensor_read_failed_disc()); + assert_eq!(discriminant(&err), tensor_read_failed_disc()); } #[test] @@ -893,14 +891,14 @@ mod tests { .write("read_tensor_f32_reports_a_file_removed_after_opening"); let context = GgufContext::from_file(&fixture.path).unwrap(); - std::fs::remove_file(&fixture.path).unwrap(); + fs::remove_file(&fixture.path).unwrap(); let result = context.read_tensor_f32("synthetic.weight"); - std::fs::write(&fixture.path, b"").unwrap(); + fs::write(&fixture.path, b"").unwrap(); assert_eq!( - std::mem::discriminant(&result.unwrap_err()), + discriminant(&result.unwrap_err()), tensor_read_failed_disc() ); } @@ -910,7 +908,7 @@ mod tests { let context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.read_tensor_f32("foo\0bar").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), nul_error_disc()); + assert_eq!(discriminant(&err), nul_error_disc()); } #[test] @@ -934,7 +932,7 @@ mod tests { let index = context.find_key("general.architecture").unwrap(); let err = context.val_arr_i32(index).unwrap_err(); - assert_eq!(std::mem::discriminant(&err), value_type_mismatch_disc()); + assert_eq!(discriminant(&err), value_type_mismatch_disc()); } #[test] @@ -951,10 +949,7 @@ mod tests { let array_index = context.find_key("synthetic.u8_array").unwrap(); let err = context.val_arr_i32(array_index).unwrap_err(); - assert_eq!( - std::mem::discriminant(&err), - array_element_type_mismatch_disc() - ); + assert_eq!(discriminant(&err), array_element_type_mismatch_disc()); } #[test] @@ -962,7 +957,7 @@ mod tests { let mut context = GgufContext::from_file(fixture_path()).unwrap(); let err = context.remove_key("nonexistent.key").unwrap_err(); - assert_eq!(std::mem::discriminant(&err), key_not_found_disc()); + assert_eq!(discriminant(&err), key_not_found_disc()); } #[test] @@ -989,11 +984,11 @@ mod tests { let removed_err = rewritten.find_key("synthetic.removed").unwrap_err(); let tensor = rewritten.read_tensor_f32("synthetic.weight").unwrap(); - std::fs::remove_file(&rewritten_path).unwrap(); + fs::remove_file(&rewritten_path).unwrap(); assert_eq!(rewritten.n_kv(), 1); assert_eq!(rewritten.val_str(kept_index).unwrap(), "kept value"); - assert_eq!(std::mem::discriminant(&removed_err), key_not_found_disc()); + assert_eq!(discriminant(&removed_err), key_not_found_disc()); assert_eq!(tensor.shape, [3, 2, 1, 1]); assert_eq!(tensor.values, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); } @@ -1021,7 +1016,7 @@ mod tests { let str_index = rewritten.find_key("synthetic.str_value").unwrap(); let array_index = rewritten.find_key("synthetic.i32_array").unwrap(); - std::fs::remove_file(&rewritten_path).unwrap(); + fs::remove_file(&rewritten_path).unwrap(); assert_eq!(rewritten.val_u32(u32_index).unwrap(), 4096); assert_eq!(rewritten.val_str(str_index).unwrap(), "set value"); @@ -1035,16 +1030,16 @@ mod tests { .write("write_to_file_reports_a_source_file_removed_after_opening"); let context = GgufContext::from_file(&fixture.path).unwrap(); - std::fs::remove_file(&fixture.path).unwrap(); + fs::remove_file(&fixture.path).unwrap(); let result = context.write_to_file(rewritten_gguf_path( "write_to_file_reports_a_source_file_removed_after_opening", )); - std::fs::write(&fixture.path, b"").unwrap(); + fs::write(&fixture.path, b"").unwrap(); assert_eq!( - std::mem::discriminant(&result.unwrap_err()), + discriminant(&result.unwrap_err()), source_file_read_failed_disc() ); } @@ -1061,7 +1056,7 @@ mod tests { .write("write_to_file_reports_a_source_file_truncated_before_its_tensor_data"); let context = GgufContext::from_file(&fixture.path).unwrap(); - std::fs::write(&fixture.path, b"").unwrap(); + fs::write(&fixture.path, b"").unwrap(); let err = context .write_to_file(rewritten_gguf_path( @@ -1069,7 +1064,7 @@ mod tests { )) .unwrap_err(); - assert_eq!(std::mem::discriminant(&err), source_file_truncated_disc()); + assert_eq!(discriminant(&err), source_file_truncated_disc()); } #[test] @@ -1079,9 +1074,6 @@ mod tests { .write_to_file("/nonexistent/directory/rewritten.gguf") .unwrap_err(); - assert_eq!( - std::mem::discriminant(&err), - destination_file_write_failed_disc() - ); + assert_eq!(discriminant(&err), destination_file_write_failed_disc()); } } diff --git a/llama-cpp-bindings/src/gguf_type.rs b/llama-cpp-bindings/src/gguf_type.rs index e59451e12..31b3fe054 100644 --- a/llama-cpp-bindings/src/gguf_type.rs +++ b/llama-cpp-bindings/src/gguf_type.rs @@ -74,7 +74,7 @@ mod tests { } #[test] - fn to_raw_roundtrips() { + fn every_known_gguf_type_converts_back_to_its_raw_value() { for raw in 0..=12 { let gguf_type = GgufType::from_raw(raw).unwrap(); assert_eq!(gguf_type.to_raw(), raw); diff --git a/llama-cpp-bindings/src/grammar_matcher.rs b/llama-cpp-bindings/src/grammar_matcher.rs index 406214372..d52a86d09 100644 --- a/llama-cpp-bindings/src/grammar_matcher.rs +++ b/llama-cpp-bindings/src/grammar_matcher.rs @@ -1,4 +1,5 @@ use std::any::Any; +use std::fmt::Display; use std::panic::AssertUnwindSafe; use std::panic::catch_unwind; @@ -6,13 +7,9 @@ use llguidance::TokenParser; use llguidance::api::StopReason; use crate::error::grammar_runtime_error::GrammarRuntimeError; +use crate::grammar_matcher_step_outcome::GrammarMatcherStepOutcome; use crate::mask_outcome::MaskOutcome; -enum StepOutcome { - Produced(TValue), - BenignStop, -} - fn panic_payload_message(payload: &(dyn Any + Send)) -> String { if let Some(message) = payload.downcast_ref::<&'static str>() { return (*message).to_owned(); @@ -25,15 +22,25 @@ fn panic_payload_message(payload: &(dyn Any + Send)) -> String { "the parser panicked with a payload that carries no message".to_owned() } +fn catch_step_panic( + operation: &'static str, + step: impl FnOnce() -> TValue, +) -> Result { + catch_unwind(AssertUnwindSafe(step)).map_err(|panic_payload| GrammarRuntimeError::Panicked { + operation, + message: panic_payload_message(panic_payload.as_ref()), + }) +} + fn stop_reason_to_result( stop_reason: StopReason, detail: String, ) -> Result<(), GrammarRuntimeError> { match stop_reason { - StopReason::NotStopped - | StopReason::NoExtension - | StopReason::NoExtensionBias - | StopReason::EndOfSentence => Ok(()), + StopReason::NoExtension | StopReason::NoExtensionBias | StopReason::EndOfSentence => Ok(()), + StopReason::NotStopped => { + Err(GrammarRuntimeError::FailedWithoutStopping { message: detail }) + } StopReason::InternalError => { Err(GrammarRuntimeError::InternalParserError { message: detail }) } @@ -76,8 +83,8 @@ impl GrammarMatcher { /// state (distinct from a benign grammar completion). pub fn compute_mask(&mut self) -> Result { match self.run("compute_mask", TokenParser::compute_mask)? { - StepOutcome::Produced(mask) => Ok(MaskOutcome::Constrained(mask)), - StepOutcome::BenignStop => Ok(MaskOutcome::GrammarComplete), + GrammarMatcherStepOutcome::Produced(mask) => Ok(MaskOutcome::Constrained(mask)), + GrammarMatcherStepOutcome::BenignStop => Ok(MaskOutcome::GrammarComplete), } } @@ -87,7 +94,9 @@ impl GrammarMatcher { /// benign stop and yields `Ok(())`. pub fn consume_token(&mut self, token: u32) -> Result<(), GrammarRuntimeError> { match self.run("consume_token", |parser| parser.consume_token(token))? { - StepOutcome::Produced(_) | StepOutcome::BenignStop => Ok(()), + GrammarMatcherStepOutcome::Produced(_) | GrammarMatcherStepOutcome::BenignStop => { + Ok(()) + } } } @@ -95,59 +104,101 @@ impl GrammarMatcher { /// Returns [`GrammarRuntimeError`] when the parser cannot be reset. pub fn reset(&mut self) -> Result<(), GrammarRuntimeError> { match self.run("reset", TokenParser::reset)? { - StepOutcome::Produced(()) | StepOutcome::BenignStop => Ok(()), + GrammarMatcherStepOutcome::Produced(()) | GrammarMatcherStepOutcome::BenignStop => { + Ok(()) + } } } - fn run( + fn run( &mut self, operation: &'static str, step: impl FnOnce(&mut TokenParser) -> Result, - ) -> Result, GrammarRuntimeError> { - match catch_unwind(AssertUnwindSafe(|| step(&mut self.parser))) { - Ok(Ok(value)) => Ok(StepOutcome::Produced(value)), - Ok(Err(step_error)) => { + ) -> Result, GrammarRuntimeError> { + match catch_step_panic(operation, || step(&mut self.parser))? { + Ok(value) => Ok(GrammarMatcherStepOutcome::Produced(value)), + Err(step_error) => { let detail = self .parser .error_message() .unwrap_or_else(|| step_error.to_string()); stop_reason_to_result(self.parser.stop_reason(), detail)?; - Ok(StepOutcome::BenignStop) + Ok(GrammarMatcherStepOutcome::BenignStop) } - Err(panic_payload) => Err(GrammarRuntimeError::Panicked { - operation, - message: panic_payload_message(panic_payload.as_ref()), - }), } } } #[cfg(test)] mod tests { + use std::any::Any; + + use llguidance::ParserFactory; + use llguidance::TokenParser; use llguidance::api::StopReason; + use llguidance::api::TopLevelGrammar; + use toktrie::ApproximateTokEnv; + use super::GrammarMatcher; use super::panic_payload_message; use super::stop_reason_to_result; use crate::error::grammar_runtime_error::GrammarRuntimeError; + fn letters_a_then_b_parser() -> TokenParser { + ParserFactory::new_simple(&ApproximateTokEnv::single_byte_env()) + .unwrap() + .create_parser(TopLevelGrammar::from_regex("ab")) + .unwrap() + } + + #[test] + fn a_started_parser_keeps_the_tokens_it_consumed() { + let mut parser = letters_a_then_b_parser(); + + parser.start_without_prompt(); + parser.consume_token(u32::from(b'a')).unwrap(); + + assert_eq!( + GrammarMatcher::new(parser).consume_token(u32::from(b'b')), + Ok(()) + ); + } + + #[test] + fn a_panicking_parser_step_is_reported_with_its_operation_and_message() { + let mut matcher = GrammarMatcher::new(letters_a_then_b_parser()); + + assert_eq!( + matcher + .run("compute_mask", |_parser| -> Result<(), String> { + panic!("parser exploded") + }) + .err(), + Some(GrammarRuntimeError::Panicked { + operation: "compute_mask", + message: "parser exploded".to_owned(), + }) + ); + } + #[test] fn a_static_str_panic_payload_is_preserved() { - let payload: Box = Box::new("parser exploded"); + let payload: Box = Box::new("parser exploded"); assert_eq!(panic_payload_message(payload.as_ref()), "parser exploded"); } #[test] fn an_owned_string_panic_payload_is_preserved() { - let payload: Box = Box::new("lexer exploded".to_owned()); + let payload: Box = Box::new("lexer exploded".to_owned()); assert_eq!(panic_payload_message(payload.as_ref()), "lexer exploded"); } #[test] fn a_payload_without_a_message_is_described() { - let payload: Box = Box::new(42_u32); + let payload: Box = Box::new(42_u32); assert_eq!( panic_payload_message(payload.as_ref()), @@ -158,15 +209,24 @@ mod tests { #[test] fn benign_stop_reasons_are_ok() { for reason in [ - StopReason::NotStopped, StopReason::NoExtension, StopReason::NoExtensionBias, StopReason::EndOfSentence, ] { - assert!(stop_reason_to_result(reason, String::new()).is_ok()); + assert_eq!(stop_reason_to_result(reason, String::new()), Ok(())); } } + #[test] + fn a_failed_step_that_did_not_stop_the_parser_is_an_error() { + assert_eq!( + stop_reason_to_result(StopReason::NotStopped, "boom".to_string()), + Err(GrammarRuntimeError::FailedWithoutStopping { + message: "boom".to_string() + }) + ); + } + #[test] fn internal_error_maps_to_internal_parser_error_with_message() { assert_eq!( diff --git a/llama-cpp-bindings/src/grammar_matcher_step_outcome.rs b/llama-cpp-bindings/src/grammar_matcher_step_outcome.rs new file mode 100644 index 000000000..39e161a95 --- /dev/null +++ b/llama-cpp-bindings/src/grammar_matcher_step_outcome.rs @@ -0,0 +1,4 @@ +pub enum GrammarMatcherStepOutcome { + Produced(TValue), + BenignStop, +} diff --git a/llama-cpp-bindings/src/json_probe_outcome.rs b/llama-cpp-bindings/src/json_probe_outcome.rs deleted file mode 100644 index a5b76fe7c..000000000 --- a/llama-cpp-bindings/src/json_probe_outcome.rs +++ /dev/null @@ -1,6 +0,0 @@ -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum JsonProbeOutcome { - StillPossiblyValid, - CompletedValid, - Failed, -} diff --git a/llama-cpp-bindings/src/json_schema_to_grammar.rs b/llama-cpp-bindings/src/json_schema_to_grammar.rs index c4cc7615c..79aeb94c2 100644 --- a/llama-cpp-bindings/src/json_schema_to_grammar.rs +++ b/llama-cpp-bindings/src/json_schema_to_grammar.rs @@ -1,7 +1,11 @@ use std::ffi::{CStr, CString, c_char}; +use std::ptr; -use crate::error::JsonSchemaToGrammarError; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + +use crate::error::json_schema_to_grammar_error::JsonSchemaToGrammarError; /// # Safety /// @@ -17,7 +21,7 @@ unsafe fn json_schema_to_grammar_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_OK => { if out.is_null() { - return Err(crate::FfiContractError { + return Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "success status contained a null grammar string", } @@ -43,6 +47,16 @@ unsafe fn json_schema_to_grammar_status_to_result( }?; Err(JsonSchemaToGrammarError::InvalidSchema { message }) } + llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_JSON => { + let message = unsafe { + read_and_free_cpp_string( + error_ptr, + "llama_rs_json_schema_to_grammar", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(JsonSchemaToGrammarError::InvalidJson { message }) + } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION => { let message = unsafe { read_and_free_cpp_string( @@ -54,27 +68,27 @@ unsafe fn json_schema_to_grammar_status_to_result( Err(JsonSchemaToGrammarError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_NULL_SCHEMA_JSON_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null schema_json argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_NULL_OUT_GRAMMAR_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null out_grammar argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_json_schema_to_grammar", code: i64::from(other), } @@ -88,13 +102,12 @@ unsafe fn json_schema_to_grammar_status_to_result( /// the wrapper reports any non-OK status, or the returned grammar is not valid UTF-8. pub fn json_schema_to_grammar(schema_json: &str) -> Result { let schema_cstr = CString::new(schema_json)?; - let mut out: *mut c_char = std::ptr::null_mut(); - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut out: *mut c_char = ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_json_schema_to_grammar( schema_cstr.as_ptr(), - false, &raw mut out, &raw mut error_ptr, ) @@ -105,77 +118,81 @@ pub fn json_schema_to_grammar(schema_json: &str) -> Result String { + if pattern.is_empty() { + return "^$".to_owned(); + } + + let start = if pattern.starts_with('^') { "" } else { "^" }; + let end = if pattern.ends_with('$') { "" } else { "$" }; + + format!("{start}{pattern}{end}") +} + +fn regex_escape_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_regex_escape_status, + out_string: *mut c_char, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_regex_escape"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK => Ok(unsafe { + read_and_free_cpp_string( + out_string, + OPERATION, + "success status contained a null string", + ) + }?), + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + Err(GrammarError::Reported { + message: unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED => { + Err(GrammarError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(GrammarError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG => Err(FfiContractError { + operation: OPERATION, + detail: "was given a null text argument", + } + .into()), + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG => { + Err(FfiContractError { + operation: OPERATION, + detail: "was given a null out_string argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG => Err(FfiContractError { + operation: OPERATION, + detail: "was given a null out_error argument", + } + .into()), + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn regex_escaped(word: &str) -> Result { + let mut out_string: *mut c_char = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_regex_escape( + word.as_ptr().cast::(), + word.len(), + &raw mut out_string, + &raw mut out_error, + ) + }; + + regex_escape_status_to_result(status, out_string, out_error) +} + +/// The trigger patterns and tokens that start a lazy grammar, built from a chat session's +/// grammar triggers the way llama.cpp's sampler builds them. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LazyGrammarTriggers { + pub patterns: Vec, + pub tokens: Vec, +} + +impl LazyGrammarTriggers { + /// # Errors + /// + /// Returns the [`GrammarError`] of escaping a trigger word. + pub fn of(triggers: Vec) -> Result { + let mut patterns = Vec::new(); + let mut tokens = Vec::new(); + + for trigger in triggers { + match trigger { + ChatGrammarTrigger::Token(token) => tokens.push(token), + ChatGrammarTrigger::Word(word) => patterns.push(regex_escaped(&word)?), + ChatGrammarTrigger::Pattern(pattern) => patterns.push(pattern), + ChatGrammarTrigger::PatternFull(pattern) => { + patterns.push(anchored_pattern(&pattern)); + } + } + } + + Ok(Self { patterns, tokens }) + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::ptr; + + use llama_cpp_fault_injection::armed_fault::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::LazyGrammarTriggers; + use super::anchored_pattern; + use super::regex_escape_status_to_result; + use crate::chat_grammar_trigger::ChatGrammarTrigger; + use crate::error::grammar_error::GrammarError; + use crate::token::LlamaToken; + + #[test] + fn regex_escape_status_to_result_maps_every_status_to_the_escaped_word_or_its_error() { + let escape = |status, out_string, out_error| { + regex_escape_status_to_result(status, out_string, out_error) + }; + let contract = |detail| { + Err(GrammarError::FfiContract(FfiContractError { + operation: "llama_rs_regex_escape", + detail, + })) + }; + + assert_eq!( + [ + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK, + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"\\[TOOL\\]".as_ptr()) }, + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_OK, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"the escape threw".as_ptr()) + } + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_TEXT_ARG, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_STRING_ARG, + ptr::null_mut(), + ptr::null_mut() + ), + escape( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_NULL_OUT_ERROR_ARG, + ptr::null_mut(), + ptr::null_mut() + ), + escape(255, ptr::null_mut(), ptr::null_mut()), + ], + [ + Ok("\\[TOOL\\]".to_owned()), + Err(GrammarError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_regex_escape", + detail: "success status contained a null string", + } + ))), + Err(GrammarError::Reported { + message: "the escape threw".to_owned() + }), + Err(GrammarError::NotEnoughMemory), + Err(GrammarError::LlamaCppOutOfMemory), + contract("was given a null text argument"), + contract("was given a null out_string argument"), + contract("was given a null out_error argument"), + Err(GrammarError::FfiStatus(FfiStatusError { + operation: "llama_rs_regex_escape", + code: 255 + })), + ] + ); + } + + #[test] + fn an_escape_failure_without_a_message_is_a_contract_violation() { + assert_eq!( + regex_escape_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_REGEX_ESCAPE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ptr::null_mut() + ), + Err(GrammarError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_regex_escape", + detail: "reported a thrown C++ exception without an error message" + } + ))) + ); + } + + #[test] + fn a_full_match_pattern_is_anchored_at_both_ends_once() { + assert_eq!( + ["", "abc", "^abc", "abc$", "^abc$"].map(anchored_pattern), + ["^$", "^abc$", "^abc$", "^abc$", "^abc$"].map(str::to_owned) + ); + } + + #[test] + fn every_trigger_kind_becomes_a_lazy_grammar_trigger() { + assert_eq!( + LazyGrammarTriggers::of(vec![ + ChatGrammarTrigger::Token(LlamaToken::new(151_657)), + ChatGrammarTrigger::Word("[TOOL_CALLS]".to_owned()), + ChatGrammarTrigger::Pattern("".to_owned()), + ChatGrammarTrigger::PatternFull("\\s*\\{".to_owned()), + ]), + Ok(LazyGrammarTriggers { + patterns: vec![ + "\\[TOOL_CALLS\\]".to_owned(), + "".to_owned(), + "^\\s*\\{$".to_owned(), + ], + tokens: vec![LlamaToken::new(151_657)], + }) + ); + } + + #[test] + fn a_trigger_word_llama_cpp_cannot_escape_is_reported() { + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_REGEX_ESCAPE_THROWS); + + assert_eq!( + LazyGrammarTriggers::of(vec![ChatGrammarTrigger::Word("".to_owned())]), + Err(GrammarError::Reported { + message: injected_fault_message(), + }) + ); + } +} diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index b3eb17b46..17f1f9578 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -3,27 +3,43 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] -pub mod bare_json_tool_calls; +mod backend_device_index; pub mod batch_add_error; +pub mod chat_assistant_prefill; +pub mod chat_assistant_turn; +pub mod chat_classification; +pub mod chat_grammar_trigger; pub mod chat_message_parse_outcome; -pub mod chat_message_parser; -pub mod chat_template_tool_calls; +pub mod chat_output; +pub mod chat_output_completeness; +pub mod chat_piece_role; +pub mod chat_prompt_part; +pub mod chat_session; +mod chat_session_attribution; +pub mod chat_session_options; +pub mod chat_span; +pub mod chat_span_kind; +mod chat_span_segment; +pub mod chat_spans; +pub mod chat_templates; +pub mod chat_tool_choice; pub mod classified_sample; pub mod context; +pub mod cpp_text_view; +pub mod dry_sequence_breakers; pub mod error; pub mod eval_multimodal_chunks_params; -pub mod extract_tool_call_markers_from_haystack; pub mod generation_progress; -pub mod ggml_time_us; pub mod gguf_context; pub mod gguf_context_error; pub mod gguf_tensor_f32; pub mod gguf_type; pub mod grammar_matcher; +mod grammar_matcher_step_outcome; pub mod ingest_outcome; pub mod invalid_numa_strategy; -pub mod json_probe_outcome; pub mod json_schema_to_grammar; +pub mod lazy_grammar_triggers; pub mod llama_backend; pub mod llama_backend_device; pub mod llama_backend_device_type; @@ -34,82 +50,23 @@ pub mod llama_token_attr; pub mod llama_token_attrs; pub mod llama_token_attrs_from_int_error; pub mod llguidance_sampler; -#[cfg(feature = "dynamic-backends")] -pub mod load_backends; -#[cfg(feature = "dynamic-backends")] pub mod load_backends_error; -#[cfg(feature = "dynamic-backends")] pub mod load_backends_from_path; pub mod log_options; pub mod log_record; -pub mod marker_role; -pub mod marker_role_candidate; pub mod mask_outcome; pub mod max_devices; -pub mod mlock_supported; -pub mod mmap_supported; pub mod model; pub mod mtmd; +pub mod parsed_chat; +mod pending_chat_piece; pub mod raw_chat_message; -pub mod resolved_tool_call_markers; pub mod sampled_token; pub mod sampled_token_classifier; pub mod sampled_token_section; pub mod sampling; pub mod sanitized_grammar; pub mod send_logs_to_log; -pub mod streaming_json_probe; -pub mod streaming_marker; -pub mod streaming_markers; -pub mod synthetic_tool_call_renders; -pub mod timing; pub mod token; +mod token_attribution; pub mod token_piece; -pub mod tool_call_format; -pub mod tool_call_marker_pair; - -pub use error::{ - ApplyChatTemplateError, ChatTemplateError, ClearKvCacheSeqError, CopyKvCacheSeqError, - DecodeError, EmbeddingsError, EncodeError, EvalMultimodalChunksError, FfiContractError, - FfiStatusError, GrammarError, JsonSchemaToGrammarError, KvCacheSeqAddError, KvCacheSeqDivError, - KvCacheSeqPosMaxError, LlamaContextLoadError, LlamaCppError, LlamaLoraAdapterInitError, - LlamaLoraAdaptersError, LlamaModelLoadError, LogitsError, MarkerDetectionError, MetaValError, - ModelParamsError, NewLlamaChatMessageError, ParseChatMessageError, Result, SampleError, - SamplerAcceptError, SamplingError, StringToTokenError, TokenAttrError, TokenOutsideVocabulary, - TokenSamplingError, TokenToStringError, -}; - -pub use bare_json_tool_calls::BareJsonToolCalls; -pub use chat_message_parse_outcome::ChatMessageParseOutcome; -pub use chat_message_parser::ChatMessageParser; -pub use classified_sample::ClassifiedSample; -pub use eval_multimodal_chunks_params::EvalMultimodalChunksParams; -pub use generation_progress::GenerationProgress; -pub use llama_backend_device::LlamaBackendDevice; -pub use llama_backend_device_type::LlamaBackendDeviceType; -pub use llama_cpp_bindings_types::{ - BracketedJsonShape, KeyValueXmlTagsShape, PairedQuoteShape, ParsedChatMessage, ParsedToolCall, - ParsedToolCallId, ReasoningMarkers, TokenUsage, TokenUsageError, ToolCallArgsShape, - ToolCallArguments, ToolCallMarkers, ToolCallValueQuote, XmlTagsShape, -}; -pub use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; -pub use marker_role::MarkerRole; -pub use marker_role_candidate::MarkerRoleCandidate; -pub use raw_chat_message::RawChatMessage; -pub use sampled_token::SampledToken; -pub use sampled_token_classifier::SampledTokenClassifier; -pub use sampled_token_section::SampledTokenSection; -pub use streaming_marker::StreamingMarker; -pub use streaming_markers::StreamingMarkers; -pub use synthetic_tool_call_renders::SyntheticToolCallRenders; -pub use token_piece::TokenPiece; - -pub use ggml_time_us::ggml_time_us; -pub use json_schema_to_grammar::json_schema_to_grammar; -pub use llama_time_us::llama_time_us; -pub use max_devices::max_devices; -pub use mlock_supported::mlock_supported; -pub use mmap_supported::mmap_supported; - -pub use log_options::LogOptions; -pub use send_logs_to_log::send_logs_to_log; diff --git a/llama-cpp-bindings/src/llama_backend.rs b/llama-cpp-bindings/src/llama_backend.rs index adf6df09c..cbbc08aab 100644 --- a/llama-cpp-bindings/src/llama_backend.rs +++ b/llama-cpp-bindings/src/llama_backend.rs @@ -1,9 +1,8 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::SeqCst; -use llama_cpp_bindings_sys::ggml_log_level; - -use crate::LlamaCppError; +use crate::error::Result; +use crate::error::llama_cpp_error::LlamaCppError; use crate::llama_backend_numa_strategy::NumaStrategy; #[derive(Eq, PartialEq, Debug)] @@ -12,7 +11,7 @@ pub struct LlamaBackend {} static LLAMA_BACKEND_INITIALIZED: AtomicBool = AtomicBool::new(false); impl LlamaBackend { - fn mark_init() -> crate::Result<()> { + fn mark_init() -> Result<()> { match LLAMA_BACKEND_INITIALIZED.compare_exchange(false, true, SeqCst, SeqCst) { Ok(_was_uninitialized) => Ok(()), Err(_was_already_initialized) => Err(LlamaCppError::BackendAlreadyInitialized), @@ -21,22 +20,27 @@ impl LlamaBackend { /// # Errors /// Returns an error if the backend was already initialized. - pub fn init() -> crate::Result { + pub fn init() -> Result { Self::mark_init()?; unsafe { llama_cpp_bindings_sys::llama_backend_init() } Ok(Self {}) } + /// Initializes the backend, then llama.cpp's NUMA support, which needs the CPU backend the + /// backend initialization loads. + /// /// # Errors /// Returns an error if the backend was already initialized. - pub fn init_numa(strategy: NumaStrategy) -> crate::Result { - Self::mark_init()?; + pub fn init_numa(strategy: NumaStrategy) -> Result { + let backend = Self::init()?; + unsafe { llama_cpp_bindings_sys::llama_numa_init( llama_cpp_bindings_sys::ggml_numa_strategy::from(strategy), ); } - Ok(Self {}) + + Ok(backend) } #[must_use] @@ -53,19 +57,6 @@ impl LlamaBackend { pub fn supports_mlock(&self) -> bool { unsafe { llama_cpp_bindings_sys::llama_supports_mlock() } } - - pub fn void_logs(&mut self) { - unsafe { - llama_cpp_bindings_sys::llama_log_set(Some(void_log), std::ptr::null_mut()); - } - } -} - -const unsafe extern "C" fn void_log( - _level: ggml_log_level, - _text: *const ::std::os::raw::c_char, - _user_data: *mut ::std::os::raw::c_void, -) { } impl Drop for LlamaBackend { @@ -80,78 +71,83 @@ mod tests { use serial_test::serial; use super::LlamaBackend; - use crate::LlamaCppError; + use crate::error::llama_cpp_error::LlamaCppError; + use crate::llama_backend_device::LlamaBackendDevice; + use crate::llama_backend_device_type::LlamaBackendDeviceType; + use crate::llama_backend_numa_strategy::NumaStrategy; #[test] - fn void_log_callback_accepts_a_log_record() { - unsafe { - super::void_log( - llama_cpp_bindings_sys::GGML_LOG_LEVEL_INFO, - c"test".as_ptr(), - std::ptr::null_mut(), - ); - } + #[serial] + fn initializing_the_backend_returns_it() { + assert_eq!(LlamaBackend::init(), Ok(LlamaBackend {})); } #[test] #[serial] - fn init_succeeds() { - let backend = LlamaBackend::init(); - assert!(backend.is_ok()); + fn initializing_the_backend_twice_reports_it_is_already_initialized() { + let _backend = LlamaBackend::init().unwrap(); + + assert_eq!( + LlamaBackend::init(), + Err(LlamaCppError::BackendAlreadyInitialized) + ); } #[test] #[serial] - fn double_init_returns_error() { + fn init_numa_reports_an_already_initialized_backend() { let _backend = LlamaBackend::init().unwrap(); - let second_err = LlamaBackend::init().unwrap_err(); assert_eq!( - std::mem::discriminant(&second_err), - std::mem::discriminant(&LlamaCppError::BackendAlreadyInitialized), - "expected BackendAlreadyInitialized, got {second_err:?}" + LlamaBackend::init_numa(NumaStrategy::Disabled), + Err(LlamaCppError::BackendAlreadyInitialized) ); } #[test] #[serial] - fn init_numa_returns_error_when_backend_already_initialized() { - use crate::llama_backend_numa_strategy::NumaStrategy; - - let _backend = LlamaBackend::init().unwrap(); - let second_err = LlamaBackend::init_numa(NumaStrategy::Disabled).unwrap_err(); + fn the_backend_supports_mmap_and_mlock() { + let backend = LlamaBackend::init().unwrap(); assert_eq!( - std::mem::discriminant(&second_err), - std::mem::discriminant(&LlamaCppError::BackendAlreadyInitialized), - "expected BackendAlreadyInitialized, got {second_err:?}" + [backend.supports_mmap(), backend.supports_mlock()], + [true, true] ); } #[test] #[serial] - fn feature_queries_return_bools() { + fn the_backend_offloads_to_a_gpu_exactly_when_one_is_registered() { let backend = LlamaBackend::init().unwrap(); - let _gpu = backend.supports_gpu_offload(); - let _mmap = backend.supports_mmap(); - let _mlock = backend.supports_mlock(); + let offloads_to_a_gpu = backend.supports_gpu_offload(); + + assert_eq!( + offloads_to_a_gpu, + LlamaBackendDevice::list_all() + .unwrap() + .iter() + .any(|device| matches!( + device.device_type, + LlamaBackendDeviceType::Gpu | LlamaBackendDeviceType::IntegratedGpu + )) + ); } #[test] #[serial] - fn dropping_backend_allows_reinitialization() { + fn dropping_the_backend_allows_reinitializing_it() { let backend = LlamaBackend::init().unwrap(); drop(backend); - let backend = LlamaBackend::init(); - assert!(backend.is_ok()); + + assert_eq!(LlamaBackend::init(), Ok(LlamaBackend {})); } #[test] #[serial] - fn init_numa_succeeds() { - use crate::llama_backend_numa_strategy::NumaStrategy; - - let backend = LlamaBackend::init_numa(NumaStrategy::Disabled); - assert!(backend.is_ok()); + fn init_numa_initializes_the_backend_with_a_numa_strategy() { + assert_eq!( + LlamaBackend::init_numa(NumaStrategy::Distribute), + Ok(LlamaBackend {}) + ); } } diff --git a/llama-cpp-bindings/src/llama_backend_device.rs b/llama-cpp-bindings/src/llama_backend_device.rs index b1108ed17..b91465451 100644 --- a/llama-cpp-bindings/src/llama_backend_device.rs +++ b/llama-cpp-bindings/src/llama_backend_device.rs @@ -1,14 +1,21 @@ +use std::ffi::CStr; use std::ffi::c_char; +use std::mem; -pub use crate::llama_backend_device_type::LlamaBackendDeviceType; -fn cstr_to_string(ptr: *const c_char) -> String { - if ptr.is_null() { - String::new() - } else { - unsafe { std::ffi::CStr::from_ptr(ptr) } - .to_string_lossy() - .to_string() +use crate::error::backend_device_text_error::BackendDeviceTextError; +use crate::error::llama_backend_device_error::LlamaBackendDeviceError; +use crate::llama_backend_device_type::LlamaBackendDeviceType; + +fn text_of(text: *const c_char) -> Result { + if text.is_null() { + return Err(BackendDeviceTextError::Null); } + + Ok(unsafe { CStr::from_ptr(text) }.to_str()?.to_owned()) +} + +fn device_text(text: *const c_char) -> Result { + text_of(unsafe { llama_cpp_bindings_sys::llama_rs_backend_device_text(text) }) } #[derive(Debug, Clone)] @@ -23,69 +30,85 @@ pub struct LlamaBackendDevice { } impl LlamaBackendDevice { - #[must_use] - pub fn list_all() -> Vec { - let mut devices = Vec::new(); - let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; - - for device_index in 0..device_count { - let dev = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; - let props = unsafe { - let mut props = std::mem::zeroed(); - llama_cpp_bindings_sys::ggml_backend_dev_get_props(dev, &raw mut props); - props - }; - let name = cstr_to_string(props.name); - let description = cstr_to_string(props.description); - let backend_reg = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_backend_reg(dev) }; - let backend_name = - unsafe { llama_cpp_bindings_sys::ggml_backend_reg_name(backend_reg) }; - let backend = cstr_to_string(backend_name); - let memory_total = props.memory_total; - let memory_free = props.memory_free; - let device_type = LlamaBackendDeviceType::from_raw(props.type_); - devices.push(Self { - index: device_index, - name, - description, - backend, - memory_total, - memory_free, - device_type, - }); - } - - devices + fn at_index(device_index: usize) -> Result { + let dev = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; + let props = unsafe { + let mut props = mem::zeroed(); + llama_cpp_bindings_sys::ggml_backend_dev_get_props(dev, &raw mut props); + props + }; + let backend_reg = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_backend_reg(dev) }; + + Ok(Self { + index: device_index, + name: device_text(props.name).map_err(LlamaBackendDeviceError::Name)?, + description: device_text(props.description) + .map_err(LlamaBackendDeviceError::Description)?, + backend: device_text(unsafe { + llama_cpp_bindings_sys::ggml_backend_reg_name(backend_reg) + }) + .map_err(LlamaBackendDeviceError::BackendName)?, + memory_total: props.memory_total, + memory_free: props.memory_free, + device_type: LlamaBackendDeviceType::try_from(unsafe { + llama_cpp_bindings_sys::llama_rs_backend_device_type(dev) + })?, + }) + } + + /// Every device ggml registered, in registration order. + /// + /// # Errors + /// + /// Returns [`LlamaBackendDeviceError`] when ggml reports a device text that is missing or not + /// UTF-8, or a device type it does not define. + pub fn list_all() -> Result, LlamaBackendDeviceError> { + (0..unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }) + .map(Self::at_index) + .collect() } } #[cfg(test)] mod tests { - use super::LlamaBackendDevice; - use super::cstr_to_string; + use std::ptr; - #[test] - fn cstr_to_string_with_null_returns_empty() { - let result = cstr_to_string(std::ptr::null()); + use super::text_of; + use crate::error::backend_device_text_error::BackendDeviceTextError; + use crate::llama_backend_device::LlamaBackendDevice; - assert_eq!(result, ""); + #[test] + fn missing_device_text_is_refused() { + assert_eq!(text_of(ptr::null()), Err(BackendDeviceTextError::Null)); } #[test] - fn cstr_to_string_with_valid_ptr() { - let result = cstr_to_string(c"hello".as_ptr()); + fn device_text_that_is_not_utf8_is_refused() { + let not_utf8 = c"\xff"; + + assert_eq!( + text_of(not_utf8.as_ptr()), + Err(BackendDeviceTextError::NotUtf8( + not_utf8.to_str().expect_err("0xff is not UTF-8") + )) + ); + } - assert_eq!(result, "hello"); + #[test] + fn device_text_is_copied() { + assert_eq!(text_of(c"CUDA0".as_ptr()), Ok("CUDA0".to_owned())); } #[test] - fn list_devices_returns_at_least_one() { - #[cfg(feature = "dynamic-backends")] - crate::load_backends::load_backends().unwrap(); - - let devices = LlamaBackendDevice::list_all(); - assert!(!devices.is_empty()); - assert_eq!(devices[0].index, 0); - assert_ne!(devices[0].name, ""); + fn every_registered_device_is_listed_at_its_index() { + let devices = LlamaBackendDevice::list_all().unwrap(); + + assert_eq!( + devices + .iter() + .map(|device| device.index) + .collect::>(), + (0..devices.len()).collect::>() + ); } } diff --git a/llama-cpp-bindings/src/llama_backend_device_type.rs b/llama-cpp-bindings/src/llama_backend_device_type.rs index c1f971e70..779828d8e 100644 --- a/llama-cpp-bindings/src/llama_backend_device_type.rs +++ b/llama-cpp-bindings/src/llama_backend_device_type.rs @@ -1,52 +1,60 @@ +use crate::error::unknown_backend_device_type::UnknownBackendDeviceType; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LlamaBackendDeviceType { Cpu, - Accelerator, Gpu, IntegratedGpu, - Unknown, + Accelerator, + Meta, } -impl LlamaBackendDeviceType { - #[must_use] - pub const fn from_raw(raw_type: llama_cpp_bindings_sys::ggml_backend_dev_type) -> Self { - match raw_type { - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU => Self::Cpu, - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL => Self::Accelerator, - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU => Self::Gpu, - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU => Self::IntegratedGpu, - _ => Self::Unknown, +impl TryFrom for LlamaBackendDeviceType { + type Error = UnknownBackendDeviceType; + + fn try_from(value: llama_cpp_bindings_sys::ggml_backend_dev_type) -> Result { + match value { + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU => Ok(Self::Cpu), + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU => Ok(Self::Gpu), + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU => Ok(Self::IntegratedGpu), + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL => Ok(Self::Accelerator), + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_META => Ok(Self::Meta), + unknown => Err(UnknownBackendDeviceType(unknown)), } } } #[cfg(test)] mod tests { - use super::LlamaBackendDeviceType; + use crate::error::unknown_backend_device_type::UnknownBackendDeviceType; + use crate::llama_backend_device_type::LlamaBackendDeviceType; #[test] - fn device_type_from_raw_all_variants() { + fn every_ggml_device_type_converts_to_its_variant() { assert_eq!( - LlamaBackendDeviceType::from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU), - LlamaBackendDeviceType::Cpu - ); - assert_eq!( - LlamaBackendDeviceType::from_raw( - llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL - ), - LlamaBackendDeviceType::Accelerator - ); - assert_eq!( - LlamaBackendDeviceType::from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU), - LlamaBackendDeviceType::Gpu - ); - assert_eq!( - LlamaBackendDeviceType::from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU), - LlamaBackendDeviceType::IntegratedGpu + [ + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU, + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU, + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU, + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL, + llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_META, + ] + .map(LlamaBackendDeviceType::try_from), + [ + Ok(LlamaBackendDeviceType::Cpu), + Ok(LlamaBackendDeviceType::Gpu), + Ok(LlamaBackendDeviceType::IntegratedGpu), + Ok(LlamaBackendDeviceType::Accelerator), + Ok(LlamaBackendDeviceType::Meta), + ] ); + } + + #[test] + fn a_value_ggml_does_not_define_is_refused() { assert_eq!( - LlamaBackendDeviceType::from_raw(9999), - LlamaBackendDeviceType::Unknown + LlamaBackendDeviceType::try_from(9999), + Err(UnknownBackendDeviceType(9999)) ); } } diff --git a/llama-cpp-bindings/src/llama_backend_numa_strategy.rs b/llama-cpp-bindings/src/llama_backend_numa_strategy.rs index fccef05aa..b0044917c 100644 --- a/llama-cpp-bindings/src/llama_backend_numa_strategy.rs +++ b/llama-cpp-bindings/src/llama_backend_numa_strategy.rs @@ -42,7 +42,7 @@ mod tests { use crate::invalid_numa_strategy::InvalidNumaStrategy; #[test] - fn numa_from_and_to() { + fn every_numa_strategy_converts_to_ggml_and_back() { let numas = [ NumaStrategy::Disabled, NumaStrategy::Distribute, diff --git a/llama-cpp-bindings/src/llama_batch.rs b/llama-cpp-bindings/src/llama_batch.rs index e7c312742..9c5e8e710 100644 --- a/llama-cpp-bindings/src/llama_batch.rs +++ b/llama-cpp-bindings/src/llama_batch.rs @@ -1,154 +1,203 @@ -use std::marker::PhantomData; +use std::ffi::c_char; +use std::ptr; +use std::ptr::NonNull; -use llama_cpp_bindings_sys::{ - llama_batch, llama_batch_free, llama_batch_init, llama_pos, llama_seq_id, -}; - -use llama_cpp_bindings_types::BatchCapacity; +use llama_cpp_bindings_sys::{llama_pos, llama_rs_batch_entry, llama_seq_id}; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::batch_add_error::BatchAddError; +use crate::context::context_limits::ContextLimits; +use crate::error::batch_transfer_error::BatchTransferError; use crate::sampled_token::SampledToken; use crate::token::LlamaToken; -fn checked_n_tokens_plus_one_as_usize(n_tokens: i32) -> Result { - let incremented = n_tokens - .checked_add(1) - .ok_or(BatchAddError::TokenCountOverflow { n_tokens })?; - - checked_i32_as_usize(incremented, "the incremented token count") -} - -fn checked_i32_as_usize( - value: i32, - value_description: &'static str, -) -> Result { - usize::try_from(value).map_err(|source| BatchAddError::IntegerOverflow { - value_description, - target_type: "usize", - source, - }) -} - -fn checked_usize_as_llama_seq_id( - value: usize, - value_description: &'static str, -) -> Result { - llama_seq_id::try_from(value).map_err(|source| BatchAddError::IntegerOverflow { - value_description, - target_type: "llama_seq_id", - source, - }) -} +fn batch_ext_assign_entries_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_batch_ext_assign_entries_status, + out_llama_cpp_return_code: i32, + out_error: *mut c_char, +) -> Result<(), BatchTransferError> { + let refused_validated_entry = |detail: &'static str| -> Result<(), BatchTransferError> { + Err(FfiContractError { + operation: "llama_rs_batch_ext_assign_entries", + detail, + } + .into()) + }; -fn checked_usize_as_i32( - value: usize, - value_description: &'static str, -) -> Result { - i32::try_from(value).map_err(|source| BatchAddError::IntegerOverflow { - value_description, - target_type: "i32", - source, - }) + match status { + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED => { + Err(BatchTransferError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY => { + Err(BatchTransferError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_batch_ext_assign_entries", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(BatchTransferError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL => refused_validated_entry( + "refused an entry of a batch validated against its limits as full", + ), + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN => { + refused_validated_entry("refused a token validated against the vocabulary") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID => { + refused_validated_entry("refused a sequence id validated against the context") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE => { + Err(FfiStatusError { + operation: "llama_batch_ext_add_token", + code: i64::from(out_llama_cpp_return_code), + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED => { + refused_validated_entry("refused the position of an entry it had just added") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED => { + refused_validated_entry("refused the output flag of an entry it had just added") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS => { + refused_validated_entry("was given an entry without sequence ids") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE => { + refused_validated_entry( + "was given an entry whose sequence ids lie outside the sequence id buffer", + ) + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG => { + refused_validated_entry("was given a null entries argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG => { + refused_validated_entry("was given a null batch argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG => { + refused_validated_entry("was given a null seq_ids argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG => { + refused_validated_entry("was given a null out_llama_cpp_return_code argument") + } + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG => { + refused_validated_entry("was given a null out_error argument") + } + other => Err(FfiStatusError { + operation: "llama_rs_batch_ext_assign_entries", + code: i64::from(other), + } + .into()), + } } -fn checked_usize_as_llama_pos( - value: usize, - value_description: &'static str, -) -> Result { - llama_pos::try_from(value).map_err(|source| BatchAddError::IntegerOverflow { - value_description, - target_type: "llama_pos", - source, - }) +/// Tokens to decode together. Each entry names its sequences by a range of `seq_ids`, the layout +/// llama.cpp receives the whole batch in, in one call. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LlamaBatch { + limits: ContextLimits, + entries: Vec, + seq_ids: Vec, + output_token_indices: Vec, + n_tokens: i32, } -#[derive(Debug)] -pub struct LlamaBatch<'tokens> { - allocated: usize, - pub initialized_logits: Vec, - pub llama_batch: llama_batch, - n_seq_max: i32, - phantom: PhantomData<&'tokens [LlamaToken]>, -} +impl LlamaBatch { + #[must_use] + pub const fn new(limits: ContextLimits) -> Self { + Self { + limits, + entries: Vec::new(), + seq_ids: Vec::new(), + output_token_indices: Vec::new(), + n_tokens: 0, + } + } -impl<'tokens> LlamaBatch<'tokens> { pub fn clear(&mut self) { - self.llama_batch.n_tokens = 0; - self.initialized_logits.clear(); + self.entries.clear(); + self.seq_ids.clear(); + self.output_token_indices.clear(); + self.n_tokens = 0; } /// # Errors /// - /// Returns an error if there is insufficient space in the buffer, if the token belongs to more - /// sequences than the batch was created for, or if integer conversions fail. + /// Returns [`BatchAddError`] when the batch already holds as many tokens as one decode of the + /// context takes, the token is outside the vocabulary, `seq_ids` is empty or names a sequence + /// the context does not have, or the token count would overflow. pub fn add( &mut self, sampled_token: &SampledToken, pos: llama_pos, - seq_ids: &[i32], + seq_ids: &[llama_seq_id], logits: bool, ) -> Result<(), BatchAddError> { - let (SampledToken::Content(LlamaToken(id)) - | SampledToken::Reasoning(LlamaToken(id)) - | SampledToken::ToolCall(LlamaToken(id)) - | SampledToken::Undeterminable(LlamaToken(id))) = *sampled_token; - let required = checked_n_tokens_plus_one_as_usize(self.n_tokens())?; - - if self.allocated < required { - return Err(BatchAddError::InsufficientSpace(self.allocated)); + if self.n_tokens.unsigned_abs() >= self.limits.n_batch { + return Err(BatchAddError::BatchFull { + n_batch: self.limits.n_batch, + }); } - let offset = self.llama_batch.n_tokens; - let offset_usize = checked_i32_as_usize(offset, "n_tokens")?; - let n_seq_id = checked_usize_as_llama_seq_id(seq_ids.len(), "seq_ids.len()")?; + let token = self + .limits + .ensure_token_in_vocabulary(sampled_token.token())?; - if n_seq_id > self.n_seq_max { - return Err(BatchAddError::TooManySequenceIds { - sequence_ids: n_seq_id, - n_seq_max: self.n_seq_max, - }); + if seq_ids.is_empty() { + return Err(BatchAddError::NoSequenceIds); } - unsafe { - self.llama_batch.token.add(offset_usize).write(id); - self.llama_batch.pos.add(offset_usize).write(pos); - self.llama_batch.n_seq_id.add(offset_usize).write(n_seq_id); - for (seq_index, seq_id) in seq_ids.iter().enumerate() { - let tmp = *self.llama_batch.seq_id.add(offset_usize); - tmp.add(seq_index).write(*seq_id); - } - self.llama_batch - .logits - .add(offset_usize) - .write(i8::from(logits)); + for seq_id in seq_ids { + self.limits.ensure_sequence_id(*seq_id)?; } + let token_index = self.n_tokens; + self.n_tokens = token_index + .checked_add(1) + .ok_or(BatchAddError::TokenCountOverflow { + n_tokens: token_index, + })?; + if logits { - self.initialized_logits.push(offset); + self.output_token_indices.push(token_index); } - self.llama_batch.n_tokens += 1; + self.entries.push(llama_rs_batch_entry { + token: token.0, + pos, + seq_ids_offset: self.seq_ids.len(), + seq_ids_count: seq_ids.len(), + output: logits, + }); + self.seq_ids.extend_from_slice(seq_ids); Ok(()) } + /// Adds `tokens` at positions counting from zero in `seq_id`, with logits for the last one, + /// or for every one when `logits_all` is set. + /// /// # Errors /// - /// Returns an error if there is insufficient space in the buffer or if integer conversions fail. + /// Returns the [`BatchAddError`] of the first token [`Self::add`] refuses. pub fn add_sequence( &mut self, tokens: &[LlamaToken], - seq_id: i32, + seq_id: llama_seq_id, logits_all: bool, ) -> Result<(), BatchAddError> { - let last_index = checked_usize_as_llama_pos(tokens.len().saturating_sub(1), "n_tokens")?; - - for (position, token) in (0..).zip(tokens.iter()) { + for ((token_index, token), pos) in tokens.iter().enumerate().zip(0..) { self.add( &SampledToken::Content(*token), - position, + pos, &[seq_id], - logits_all || position == last_index, + logits_all || token_index + 1 == tokens.len(), )?; } @@ -156,462 +205,373 @@ impl<'tokens> LlamaBatch<'tokens> { } #[must_use] - pub fn new(capacity: BatchCapacity, n_seq_max: i32) -> Self { - let batch = unsafe { llama_batch_init(capacity.tokens_i32(), 0, n_seq_max) }; - - LlamaBatch { - allocated: capacity.tokens_usize(), - initialized_logits: vec![], - llama_batch: batch, - n_seq_max, - phantom: PhantomData, - } + pub const fn limits(&self) -> ContextLimits { + self.limits + } + + #[must_use] + pub const fn n_tokens(&self) -> i32 { + self.n_tokens } + /// Hands every token of this batch to llama.cpp's `batch_ext` in one call, replacing the + /// tokens it held. + /// + /// # Safety + /// + /// `batch_ext` must point to a batch that llama.cpp initialized and has not freed. + /// /// # Errors /// - /// Returns an error if the provided token buffer is empty or if integer conversions fail. - pub fn get_one(tokens: &'tokens [LlamaToken]) -> Result { - if tokens.is_empty() { - return Err(BatchAddError::EmptyBuffer); - } - - let token_count = checked_usize_as_i32(tokens.len(), "token count")?; - - let batch = unsafe { - let ptr = tokens.as_ptr().cast::().cast_mut(); - llama_cpp_bindings_sys::llama_batch_get_one(ptr, token_count) + /// Returns [`BatchTransferError`] when llama.cpp refuses an entry or the wrapper fails. + pub unsafe fn assign_to( + &self, + batch_ext: NonNull, + ) -> Result<(), BatchTransferError> { + let mut out_llama_cpp_return_code: i32 = 0; + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_batch_ext_assign_entries( + batch_ext.as_ptr(), + self.entries.as_ptr(), + self.entries.len(), + self.seq_ids.as_ptr(), + self.seq_ids.len(), + &raw mut out_llama_cpp_return_code, + &raw mut out_error, + ) }; - let last_token_index = checked_usize_as_i32(tokens.len() - 1, "last token index")?; - - Ok(Self { - allocated: 0, - initialized_logits: vec![last_token_index], - llama_batch: batch, - n_seq_max: 0, - phantom: PhantomData, - }) + batch_ext_assign_entries_status_to_result(status, out_llama_cpp_return_code, out_error) } #[must_use] - pub const fn n_tokens(&self) -> i32 { - self.llama_batch.n_tokens - } -} - -impl Drop for LlamaBatch<'_> { - fn drop(&mut self) { - unsafe { - if self.allocated > 0 { - llama_batch_free(self.llama_batch); - } - } + pub fn output_token_indices(&self) -> &[i32] { + &self.output_token_indices } } #[cfg(test)] mod tests { - use llama_cpp_bindings_types::BatchCapacity; + use llama_cpp_bindings_sys::llama_rs_batch_entry; + use super::LlamaBatch; + use crate::batch_add_error::BatchAddError; + use crate::context::context_limits::ContextLimits; + use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; use crate::sampled_token::SampledToken; use crate::token::LlamaToken; - use super::{ - BatchAddError, LlamaBatch, checked_i32_as_usize, checked_n_tokens_plus_one_as_usize, - checked_usize_as_i32, checked_usize_as_llama_pos, checked_usize_as_llama_seq_id, + const LIMITS: ContextLimits = ContextLimits { + n_batch: 4, + n_ubatch: 4, + n_seq_max: 2, + n_vocab: 100, }; - #[test] - fn new_creates_empty_batch() { - let batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - - assert_eq!(batch.n_tokens(), 0); - assert_eq!(batch.initialized_logits, Vec::::new()); + fn content(token_id: i32) -> SampledToken { + SampledToken::Content(LlamaToken::new(token_id)) } #[test] - fn clear_resets_batch() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], true) - .unwrap(); - assert_eq!(batch.n_tokens(), 1); - - batch.clear(); + fn a_batch_whose_token_count_reached_its_type_limit_refuses_another_token() { + let mut batch = LlamaBatch { + n_tokens: i32::MAX, + ..LlamaBatch::new(ContextLimits { + n_batch: u32::MAX, + ..LIMITS + }) + }; - assert_eq!(batch.n_tokens(), 0); - assert_eq!(batch.initialized_logits, Vec::::new()); + assert_eq!( + batch.add(&content(1), 0, &[0], false), + Err(BatchAddError::TokenCountOverflow { n_tokens: i32::MAX }) + ); } #[test] - fn add_increments_token_count() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); + fn added_tokens_are_kept_in_order_with_their_outputs() { + let mut batch = LlamaBatch::new(LIMITS); - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false) - .unwrap(); - assert_eq!(batch.n_tokens(), 1); + batch.add(&content(1), 0, &[0], false).unwrap(); + batch.add(&content(2), 1, &[0, 1], true).unwrap(); - batch - .add(&SampledToken::Content(LlamaToken::new(2)), 1, &[0], false) - .unwrap(); + assert_eq!( + batch.entries, + [ + llama_rs_batch_entry { + token: 1, + pos: 0, + seq_ids_offset: 0, + seq_ids_count: 1, + output: false, + }, + llama_rs_batch_entry { + token: 2, + pos: 1, + seq_ids_offset: 1, + seq_ids_count: 2, + output: true, + }, + ] + ); + assert_eq!(batch.seq_ids, [0, 0, 1]); + assert_eq!(batch.output_token_indices(), [1]); assert_eq!(batch.n_tokens(), 2); } #[test] - fn add_tracks_logits() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); + fn clearing_forgets_every_token_and_output() { + let mut batch = LlamaBatch::new(LIMITS); + batch.add(&content(1), 0, &[0], true).unwrap(); - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false) - .unwrap(); - assert_eq!(batch.initialized_logits, Vec::::new()); + batch.clear(); - batch - .add(&SampledToken::Content(LlamaToken::new(2)), 1, &[0], true) - .unwrap(); - assert_eq!(batch.initialized_logits, vec![1]); + assert_eq!(batch, LlamaBatch::new(LIMITS)); } #[test] - fn add_returns_insufficient_space_when_full() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(1).unwrap(), 1); + fn a_batch_holding_one_decode_of_tokens_refuses_another() { + let mut batch = LlamaBatch::new(LIMITS); batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false) + .add_sequence(&[1, 2, 3, 4].map(LlamaToken::new), 0, false) .unwrap(); - let result = batch.add(&SampledToken::Content(LlamaToken::new(2)), 1, &[0], false); - - assert_eq!(result, Err(BatchAddError::InsufficientSpace(1))); - } - - #[test] - fn add_refuses_more_sequence_ids_than_the_batch_holds_per_token() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - - let result = batch.add( - &SampledToken::Content(LlamaToken::new(1)), - 0, - &[0, 1], - false, - ); - assert_eq!( - result, - Err(BatchAddError::TooManySequenceIds { - sequence_ids: 2, - n_seq_max: 1, - }) + batch.add(&content(5), 4, &[0], false), + Err(BatchAddError::BatchFull { n_batch: 4 }) ); - assert_eq!(batch.n_tokens(), 0); } #[test] - fn add_accepts_reasoning_sampled_token_variant() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(4).unwrap(), 1); - - batch - .add(&SampledToken::Reasoning(LlamaToken::new(11)), 0, &[0], true) - .unwrap(); - - assert_eq!(batch.n_tokens(), 1); - } - - #[test] - fn add_accepts_tool_call_sampled_token_variant() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(4).unwrap(), 1); - - batch - .add(&SampledToken::ToolCall(LlamaToken::new(22)), 0, &[0], true) - .unwrap(); - - assert_eq!(batch.n_tokens(), 1); - } + fn a_token_outside_the_vocabulary_is_refused() { + let mut batch = LlamaBatch::new(LIMITS); - #[test] - fn add_accepts_undeterminable_sampled_token_variant() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(4).unwrap(), 1); - - batch - .add( - &SampledToken::Undeterminable(LlamaToken::new(33)), - 0, - &[0], - false, - ) - .unwrap(); - - assert_eq!(batch.n_tokens(), 1); - } - - #[test] - fn add_sequence_adds_all_tokens() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - let tokens = vec![ - LlamaToken::new(10), - LlamaToken::new(20), - LlamaToken::new(30), - ]; - - batch.add_sequence(&tokens, 0, false).unwrap(); - - assert_eq!(batch.n_tokens(), 3); - } - - #[test] - fn add_sequence_sets_logits_on_last_token() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - let tokens = vec![ - LlamaToken::new(10), - LlamaToken::new(20), - LlamaToken::new(30), - ]; - - batch.add_sequence(&tokens, 0, false).unwrap(); - - assert_eq!(batch.initialized_logits, vec![2]); - } - - #[test] - fn add_sequence_insufficient_space() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2).unwrap(), 1); - let tokens = vec![ - LlamaToken::new(10), - LlamaToken::new(20), - LlamaToken::new(30), - ]; - - let result = batch.add_sequence(&tokens, 0, false); - - assert!(result.is_err()); - } - - #[test] - fn add_sequence_fails_mid_loop_when_batch_fills() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(2).unwrap(), 1); - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false) - .unwrap(); - - let tokens = vec![LlamaToken::new(10), LlamaToken::new(20)]; - let result = batch.add_sequence(&tokens, 0, false); - - assert!(result.is_err()); - } - - #[test] - fn get_one_with_valid_tokens() { - let tokens = vec![LlamaToken::new(1), LlamaToken::new(2)]; - let batch = LlamaBatch::get_one(&tokens).expect("test: get_one should succeed"); - - assert_eq!(batch.n_tokens(), 2); - assert_eq!(batch.initialized_logits, vec![1]); + assert_eq!( + batch.add(&content(100), 0, &[0], false), + Err(BatchAddError::TokenOutsideVocabulary( + TokenOutsideVocabulary { + token: LlamaToken::new(100), + n_vocab: 100, + } + )) + ); } #[test] - fn get_one_empty_slice_returns_error() { - let tokens: Vec = vec![]; - let result = LlamaBatch::get_one(&tokens); + fn a_token_without_sequences_is_refused() { + let mut batch = LlamaBatch::new(LIMITS); - assert_eq!(result.unwrap_err(), BatchAddError::EmptyBuffer); + assert_eq!( + batch.add(&content(1), 0, &[], false), + Err(BatchAddError::NoSequenceIds) + ); } #[test] - fn get_one_single_token() { - let tokens = vec![LlamaToken::new(42)]; - let batch = LlamaBatch::get_one(&tokens).expect("test: get_one should succeed"); + fn a_token_in_a_sequence_the_context_lacks_is_refused_and_not_added() { + let mut batch = LlamaBatch::new(LIMITS); - assert_eq!(batch.n_tokens(), 1); - assert_eq!(batch.initialized_logits, vec![0]); + assert_eq!( + batch.add(&content(1), 0, &[0, 2], false), + Err(BatchAddError::SequenceIdOutOfRange(SequenceIdOutOfRange { + seq_id: 2, + n_seq_max: 2, + })) + ); + assert_eq!(batch, LlamaBatch::new(LIMITS)); } #[test] - fn add_with_logits_false_retains_only_previous_logits() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - - batch - .add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], true) - .unwrap(); - assert_eq!(batch.initialized_logits, vec![0]); + fn a_sequence_requests_logits_for_its_last_token_only() { + let mut batch = LlamaBatch::new(LIMITS); batch - .add(&SampledToken::Content(LlamaToken::new(2)), 0, &[0], false) + .add_sequence(&[10, 20, 30].map(LlamaToken::new), 1, false) .unwrap(); - assert_eq!(batch.initialized_logits, vec![0]); - } - - #[test] - fn add_sequence_with_logits_all_marks_every_token() -> Result<(), BatchAddError> { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - let tokens = vec![ - LlamaToken::new(10), - LlamaToken::new(20), - LlamaToken::new(30), - ]; - batch.add_sequence(&tokens, 0, true)?; - - assert_eq!(batch.n_tokens(), 3); - assert_eq!(batch.initialized_logits, vec![0, 1, 2]); - - Ok(()) - } - - #[test] - fn add_with_multiple_seq_ids() -> Result<(), BatchAddError> { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 4); - - batch.add( - &SampledToken::Content(LlamaToken::new(1)), - 0, - &[0, 1, 2], - true, - )?; - - assert_eq!(batch.n_tokens(), 1); - assert_eq!(batch.initialized_logits, vec![0]); - - Ok(()) - } - - #[test] - fn drop_does_not_free_get_one_batch() { - let tokens = vec![LlamaToken::new(1), LlamaToken::new(2)]; - let batch = LlamaBatch::get_one(&tokens).expect("test: get_one should succeed"); - - assert_eq!(batch.allocated, 0); - drop(batch); - } - - #[test] - fn checked_n_tokens_plus_one_as_usize_succeeds_for_zero() { - let result = checked_n_tokens_plus_one_as_usize(0); - - assert_eq!(result, Ok(1)); - } - - #[test] - fn checked_n_tokens_plus_one_as_usize_fails_for_negative() { + assert_eq!(batch.output_token_indices(), [2]); assert_eq!( - checked_n_tokens_plus_one_as_usize(-2), - Err(BatchAddError::IntegerOverflow { - value_description: "the incremented token count", - target_type: "usize", - source: usize::try_from(-1_i32).expect_err("the conversion must fail"), + batch.entries.last(), + Some(&llama_rs_batch_entry { + token: 30, + pos: 2, + seq_ids_offset: 2, + seq_ids_count: 1, + output: true, }) ); } #[test] - fn checked_n_tokens_plus_one_as_usize_fails_for_i32_max() { - assert_eq!( - checked_n_tokens_plus_one_as_usize(i32::MAX), - Err(BatchAddError::TokenCountOverflow { n_tokens: i32::MAX }) - ); - } + fn a_sequence_with_logits_for_all_requests_every_token() { + let mut batch = LlamaBatch::new(LIMITS); - #[test] - fn checked_i32_as_usize_succeeds_for_zero() { - let result = checked_i32_as_usize(0, "test_value"); + batch + .add_sequence(&[10, 20, 30].map(LlamaToken::new), 0, true) + .unwrap(); - assert_eq!(result, Ok(0)); + assert_eq!(batch.output_token_indices(), [0, 1, 2]); } #[test] - fn checked_i32_as_usize_fails_for_negative() { + fn a_sequence_longer_than_one_decode_stops_at_the_first_refused_token() { + let mut batch = LlamaBatch::new(LIMITS); + assert_eq!( - checked_i32_as_usize(i32::MIN, "test_value"), - Err(BatchAddError::IntegerOverflow { - value_description: "test_value", - target_type: "usize", - source: usize::try_from(i32::MIN).expect_err("the conversion must fail"), - }) + batch.add_sequence(&[1, 2, 3, 4, 5].map(LlamaToken::new), 0, false), + Err(BatchAddError::BatchFull { n_batch: 4 }) ); + assert_eq!(batch.n_tokens(), 4); } +} - #[test] - fn checked_usize_as_llama_seq_id_succeeds_for_zero() { - let result = checked_usize_as_llama_seq_id(0, "test_value"); - - assert_eq!(result, Ok(0)); +#[cfg(test)] +mod batch_ext_assign_entries_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::ptr; + + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::batch_ext_assign_entries_status_to_result; + use crate::error::batch_transfer_error::BatchTransferError; + + #[test] + fn assigning_batch_entries_reports_each_refused_or_malformed_entry_and_null_argument_as_a_contract_violation() + { + for (status, detail) in [ + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_BATCH_FULL, + "refused an entry of a batch validated against its limits as full", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_TOKEN, + "refused a token validated against the vocabulary", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_INVALID_SEQUENCE_ID, + "refused a sequence id validated against the context", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_POSITION_REJECTED, + "refused the position of an entry it had just added", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_OUTPUT_REJECTED, + "refused the output flag of an entry it had just added", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_EMPTY_SEQ_IDS, + "was given an entry without sequence ids", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_SEQ_IDS_OUT_OF_RANGE, + "was given an entry whose sequence ids lie outside the sequence id buffer", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_ENTRIES_ARG, + "was given a null entries argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_BATCH_ARG, + "was given a null batch argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_SEQ_IDS_ARG, + "was given a null seq_ids argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_LLAMA_CPP_RETURN_CODE_ARG, + "was given a null out_llama_cpp_return_code argument", + ), + ( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_NULL_OUT_ERROR_ARG, + "was given a null out_error argument", + ), + ] { + assert_eq!( + batch_ext_assign_entries_status_to_result(status, 0, ptr::null_mut()), + Err(BatchTransferError::FfiContract(FfiContractError { + operation: "llama_rs_batch_ext_assign_entries", + detail, + })) + ); + } } #[test] - fn checked_usize_as_llama_seq_id_fails_for_overflow() { + fn an_undocumented_code_llama_cpp_returns_for_an_entry_is_reported_with_its_value() { assert_eq!( - checked_usize_as_llama_seq_id(usize::MAX, "test_value"), - Err(BatchAddError::IntegerOverflow { - value_description: "test_value", - target_type: "llama_seq_id", - source: super::llama_seq_id::try_from(usize::MAX) - .expect_err("the conversion must fail"), - }) + batch_ext_assign_entries_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_RETURNED_UNKNOWN_CODE, + -7, + ptr::null_mut(), + ), + Err(BatchTransferError::FfiStatus(FfiStatusError { + operation: "llama_batch_ext_add_token", + code: -7, + })) ); } #[test] - fn checked_usize_as_i32_succeeds_for_zero() { - let result = checked_usize_as_i32(0, "test_value"); - - assert_eq!(result, Ok(0)); - } - - #[test] - fn checked_usize_as_i32_fails_for_overflow() { + fn adding_a_batch_entry_reports_memory_exhaustion() { assert_eq!( - checked_usize_as_i32(usize::MAX, "test_value"), - Err(BatchAddError::IntegerOverflow { - value_description: "test_value", - target_type: "i32", - source: i32::try_from(usize::MAX).expect_err("the conversion must fail"), - }) + [ + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_OUT_OF_MEMORY, + ] + .map(|status| batch_ext_assign_entries_status_to_result(status, 0, ptr::null_mut())), + [ + Err(BatchTransferError::NotEnoughMemory), + Err(BatchTransferError::LlamaCppOutOfMemory), + ] ); } #[test] - fn checked_usize_as_llama_pos_succeeds_for_zero() { - let result = checked_usize_as_llama_pos(0, "test_value"); - - assert_eq!(result, Ok(0)); - } + fn adding_a_batch_entry_reports_an_exception_message() { + let out_error = unsafe { + llama_cpp_bindings_sys::llama_rs_string_dup(c"entry insertion threw".as_ptr()) + }; - #[test] - fn checked_usize_as_llama_pos_fails_for_overflow() { assert_eq!( - checked_usize_as_llama_pos(usize::MAX, "test_value"), - Err(BatchAddError::IntegerOverflow { - value_description: "test_value", - target_type: "llama_pos", - source: super::llama_pos::try_from(usize::MAX) - .expect_err("the conversion must fail"), + batch_ext_assign_entries_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ), + Err(BatchTransferError::Reported { + message: "entry insertion threw".to_owned() }) ); } #[test] - fn add_fails_when_required_token_count_overflows_i32() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - batch.llama_batch.n_tokens = i32::MAX; - + fn adding_a_batch_entry_reports_an_exception_without_a_message_as_a_contract_violation() { assert_eq!( - batch.add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false), - Err(BatchAddError::TokenCountOverflow { n_tokens: i32::MAX }) + batch_ext_assign_entries_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_BATCH_EXT_ASSIGN_ENTRIES_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + ptr::null_mut(), + ), + Err(BatchTransferError::CppString(CppStringError::Missing(FfiContractError { + operation: "llama_rs_batch_ext_assign_entries", + detail: "reported a thrown C++ exception without an error message", + }))) ); } #[test] - fn add_fails_when_existing_offset_is_negative() { - let mut batch = LlamaBatch::new(BatchCapacity::try_from(16).unwrap(), 1); - batch.llama_batch.n_tokens = -1; - + fn adding_a_batch_entry_preserves_an_unknown_status() { assert_eq!( - batch.add(&SampledToken::Content(LlamaToken::new(1)), 0, &[0], false), - Err(BatchAddError::IntegerOverflow { - value_description: "n_tokens", - target_type: "usize", - source: usize::try_from(-1_i32).expect_err("the conversion must fail"), - }) + batch_ext_assign_entries_status_to_result(255, 0, ptr::null_mut()), + Err(BatchTransferError::FfiStatus(FfiStatusError { + operation: "llama_rs_batch_ext_assign_entries", + code: 255, + })) ); } } diff --git a/llama-cpp-bindings/src/llama_time_us.rs b/llama-cpp-bindings/src/llama_time_us.rs index 63d43ad8c..1344a4783 100644 --- a/llama-cpp-bindings/src/llama_time_us.rs +++ b/llama-cpp-bindings/src/llama_time_us.rs @@ -5,10 +5,10 @@ pub fn llama_time_us() -> i64 { #[cfg(test)] mod tests { - use super::llama_time_us; + use crate::llama_time_us::llama_time_us; #[test] - fn returns_positive_value() { + fn llama_time_is_positive() { let time_microseconds = llama_time_us(); assert!(time_microseconds > 0); diff --git a/llama-cpp-bindings/src/llama_token_attrs.rs b/llama-cpp-bindings/src/llama_token_attrs.rs index d37d297ca..3f95dd36f 100644 --- a/llama-cpp-bindings/src/llama_token_attrs.rs +++ b/llama-cpp-bindings/src/llama_token_attrs.rs @@ -22,10 +22,10 @@ impl DerefMut for LlamaTokenAttrs { } } -impl TryFrom for LlamaTokenAttrs { +impl TryFrom for LlamaTokenAttrs { type Error = LlamaTokenAttrsFromIntError; - fn try_from(value: llama_cpp_bindings_sys::llama_vocab_type) -> Result { + fn try_from(value: llama_cpp_bindings_sys::llama_token_attr) -> Result { Ok(Self(BitFlags::from_bits(value).map_err( |bit_flag_error| { LlamaTokenAttrsFromIntError::UnknownValue(bit_flag_error.invalid_bits()) @@ -38,36 +38,31 @@ impl TryFrom for LlamaTokenAttrs { mod tests { use enumflags2::BitFlags; - use super::{LlamaTokenAttr, LlamaTokenAttrs, LlamaTokenAttrsFromIntError}; + use super::LlamaTokenAttr; + use super::LlamaTokenAttrs; + use super::LlamaTokenAttrsFromIntError; #[test] - fn try_from_valid_single_attribute() { - let attrs = LlamaTokenAttrs::try_from(llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_NORMAL); - - assert!(attrs.is_ok()); - assert!( - attrs - .expect("valid attribute") - .contains(LlamaTokenAttr::Normal) + fn a_known_attribute_bit_converts_to_its_flag() { + assert_eq!( + LlamaTokenAttrs::try_from(llama_cpp_bindings_sys::LLAMA_TOKEN_ATTR_NORMAL), + Ok(LlamaTokenAttrs(BitFlags::from_flag(LlamaTokenAttr::Normal))) ); } #[test] fn try_from_zero_produces_empty_flags() { - let attrs = LlamaTokenAttrs::try_from(0); - - assert!(attrs.is_ok()); - assert!(attrs.expect("valid attribute").is_empty()); + assert_eq!( + LlamaTokenAttrs::try_from(0), + Ok(LlamaTokenAttrs(BitFlags::empty())) + ); } #[test] - fn try_from_invalid_bits_returns_error() { - let err = LlamaTokenAttrs::try_from(!0).unwrap_err(); - let LlamaTokenAttrsFromIntError::UnknownValue(invalid_bits) = err; - - assert!( - invalid_bits > 0, - "passing !0 must produce at least one unknown bit" + fn unknown_attribute_bits_are_reported() { + assert_eq!( + LlamaTokenAttrs::try_from(!0), + Err(LlamaTokenAttrsFromIntError::UnknownValue(!0x3FF)) ); } diff --git a/llama-cpp-bindings/src/llguidance_sampler.rs b/llama-cpp-bindings/src/llguidance_sampler.rs index c04457adb..ffe9d2160 100644 --- a/llama-cpp-bindings/src/llguidance_sampler.rs +++ b/llama-cpp-bindings/src/llguidance_sampler.rs @@ -1,11 +1,18 @@ +use std::ffi::c_char; use std::ffi::c_void; +use std::ptr; +use std::slice; use std::sync::Arc; -use llama_cpp_error_recorder::RecordedError; -use llama_cpp_error_recorder::record; +use llguidance::api::TopLevelGrammar; +use llguidance::earley::SlicedBiasComputer; use toktrie::ApproximateTokEnv; +use toktrie::InferenceCapabilities; -use crate::GrammarError; +use llama_cpp_error_recorder::record::record; +use llama_cpp_error_recorder::recorded_error::RecordedError; + +use crate::error::grammar_error::GrammarError; use crate::grammar_matcher::GrammarMatcher; use crate::mask_outcome::MaskOutcome; use crate::model::LlamaModel; @@ -20,7 +27,7 @@ struct LlgContext { const unsafe extern "C" fn llg_name( _smpl: *const llama_cpp_bindings_sys::llama_sampler, -) -> *const std::os::raw::c_char { +) -> *const c_char { c"llguidance".as_ptr() } @@ -52,7 +59,7 @@ unsafe extern "C" fn llg_apply( } }; - let data = unsafe { std::slice::from_raw_parts_mut(cur_p.data, cur_p.size) }; + let data = unsafe { slice::from_raw_parts_mut(cur_p.data, cur_p.size) }; for item in data.iter_mut() { if !mask.is_allowed(item.id.cast_unsigned()) { item.logit = f32::NEG_INFINITY; @@ -87,10 +94,7 @@ unsafe extern "C" fn llg_clone( } unsafe extern "C" fn llg_free(smpl: *mut llama_cpp_bindings_sys::llama_sampler) { - let ctx_ptr = unsafe { (*smpl).ctx.cast::() }; - if !ctx_ptr.is_null() { - drop(unsafe { Box::from_raw(ctx_ptr) }); - } + drop(unsafe { Box::from_raw((*smpl).ctx.cast::()) }); } static mut LLG_SAMPLER_I: llama_cpp_bindings_sys::llama_sampler_i = @@ -109,26 +113,43 @@ static mut LLG_SAMPLER_I: llama_cpp_bindings_sys::llama_sampler_i = copy_state: None, }; -/// # Errors -/// -/// Returns `GrammarError` if the parser factory, grammar, or parser cannot be created. -pub fn create_llg_sampler( - model: &LlamaModel, +const UNCOMPILABLE_BIAS_SLICE: &str = "("; + +fn bias_slices() -> Vec { + let factory_fault_reached = unsafe { + llama_cpp_bindings_sys::llama_rs_fault_point_reached( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_LLGUIDANCE_FACTORY_FAILS, + ) + }; + + if factory_fault_reached { + return vec![UNCOMPILABLE_BIAS_SLICE.to_owned()]; + } + + SlicedBiasComputer::general_slices() +} + +fn create_llg_sampler_from_tok_env( + tok_env: Arc, grammar_kind: &str, grammar_data: &str, ) -> Result { - let tok_env = model.approximate_tok_env()?; let tok_env_dyn: Arc = tok_env.clone(); - let factory = llguidance::ParserFactory::new_simple(&tok_env_dyn).map_err(|factory_error| { - GrammarError::LlguidanceFactoryUnavailable { - message: factory_error.to_string(), - } + let factory = llguidance::ParserFactory::new( + &tok_env_dyn, + InferenceCapabilities::default(), + &bias_slices(), + ) + .map_err(|factory_error| GrammarError::LlguidanceFactoryUnavailable { + message: factory_error.to_string(), })?; - let grammar = llguidance::api::TopLevelGrammar::from_tagged_str(grammar_kind, grammar_data) - .map_err(|parse_error| GrammarError::LlguidanceGrammarInvalid { - message: parse_error.to_string(), + let grammar = + TopLevelGrammar::from_tagged_str(grammar_kind, grammar_data).map_err(|parse_error| { + GrammarError::LlguidanceGrammarInvalid { + message: parse_error.to_string(), + } })?; let parser = factory.create_parser(grammar).map_err(|parser_error| { @@ -147,7 +168,7 @@ pub fn create_llg_sampler( LlamaSampler::initialize("llguidance", |out_sampler, out_error| unsafe { llama_cpp_bindings_sys::llama_rs_sampler_init( &raw mut LLG_SAMPLER_I, - std::ptr::null_mut(), + ptr::null_mut(), out_sampler, out_error, ) @@ -157,3 +178,150 @@ pub fn create_llg_sampler( }) .map_err(Into::into) } + +/// # Errors +/// +/// Returns `GrammarError` if the parser factory, grammar, or parser cannot be created. +pub fn create_llg_sampler( + model: &LlamaModel, + grammar_kind: &str, + grammar_data: &str, +) -> Result { + create_llg_sampler_from_tok_env(model.approximate_tok_env()?, grammar_kind, grammar_data) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use toktrie::ApproximateTokEnv; + use toktrie::INVALID_TOKEN; + use toktrie::InferenceCapabilities; + use toktrie::TokRxInfo; + use toktrie::TokTrie; + + use llama_cpp_fault_injection::armed_fault::ArmedFault; + + use super::UNCOMPILABLE_BIAS_SLICE; + use super::create_llg_sampler_from_tok_env; + use crate::error::grammar_error::GrammarError; + use crate::error::grammar_runtime_error::GrammarRuntimeError; + use crate::error::sample_error::SampleError; + use crate::error::sampler_accept_error::SamplerAcceptError; + use crate::sampling::LlamaSampler; + use crate::token::LlamaToken; + use crate::token::data::LlamaTokenData; + use crate::token::data_array::LlamaTokenDataArray; + + const LETTER_A: LlamaToken = LlamaToken::new(0); + const LETTER_B: LlamaToken = LlamaToken::new(1); + const OUT_OF_VOCABULARY: LlamaToken = LlamaToken::new(2); + + fn letters() -> Arc { + let vocabulary_without_end_of_generation = TokRxInfo::new(2, INVALID_TOKEN); + + Arc::new(ApproximateTokEnv::new(TokTrie::from( + &vocabulary_without_end_of_generation, + &[b"a".to_vec(), b"b".to_vec()], + ))) + } + + fn letter_a_sampler() -> Result { + create_llg_sampler_from_tok_env(letters(), "regex", "a") + } + + fn letter_candidates() -> LlamaTokenDataArray { + LlamaTokenDataArray::new( + vec![ + LlamaTokenData::new(LETTER_A, 1.0, 0.0), + LlamaTokenData::new(LETTER_B, 2.0, 0.0), + ], + false, + ) + } + + fn out_of_vocabulary_failure() -> String { + GrammarRuntimeError::InternalParserError { + message: "token id 2 out of range".to_owned(), + } + .to_string() + } + + #[test] + fn apply_leaves_candidates_untouched_once_the_grammar_is_complete() { + let mut sampler = letter_a_sampler().unwrap(); + let mut candidates = letter_candidates(); + + assert_eq!(sampler.accept(LETTER_A), Ok(())); + assert_eq!(sampler.apply(&mut candidates), Ok(())); + assert_eq!(candidates, letter_candidates()); + } + + #[test] + fn apply_reports_a_grammar_in_an_error_state_as_a_callback_failure() { + let mut sampler = letter_a_sampler().unwrap(); + + assert_eq!( + sampler.accept(OUT_OF_VOCABULARY), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + assert_eq!( + sampler.apply(&mut letter_candidates()), + Err(SampleError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + } + + #[test] + fn accepting_many_tokens_stops_at_the_first_one_the_grammar_rejects() { + let mut sampler = letter_a_sampler().unwrap(); + + assert_eq!( + sampler.accept_many([OUT_OF_VOCABULARY, LETTER_A]), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + } + + #[test] + fn reset_reports_a_grammar_in_an_error_state_as_a_callback_failure() { + let mut sampler = letter_a_sampler().unwrap(); + + assert_eq!(sampler.accept(LETTER_A), Ok(())); + assert_eq!( + sampler.accept(OUT_OF_VOCABULARY), + Err(SamplerAcceptError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + assert_eq!( + sampler.reset(), + Err(SampleError::GrammarCallbackFailed { + message: out_of_vocabulary_failure(), + }) + ); + } + + #[test] + fn creating_a_sampler_reports_a_parser_factory_llguidance_cannot_build() { + let letters_env: Arc = letters(); + let factory_failure = llguidance::ParserFactory::new( + &letters_env, + InferenceCapabilities::default(), + &[UNCOMPILABLE_BIAS_SLICE.to_owned()], + ) + .err() + .map(|factory_error| GrammarError::LlguidanceFactoryUnavailable { + message: factory_error.to_string(), + }); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_LLGUIDANCE_FACTORY_FAILS); + + assert_ne!(factory_failure, None); + assert_eq!(letter_a_sampler().err(), factory_failure); + } +} diff --git a/llama-cpp-bindings/src/load_backends.rs b/llama-cpp-bindings/src/load_backends.rs deleted file mode 100644 index 54634f1a3..000000000 --- a/llama-cpp-bindings/src/load_backends.rs +++ /dev/null @@ -1,30 +0,0 @@ -use std::path::Path; - -use crate::load_backends_error::LoadBackendsError; -use crate::load_backends_from_path::load_backends_from_path; - -pub const BACKENDS_DIR: Option<&str> = option_env!("GGML_BACKENDS_DIR"); - -/// # Errors -/// -/// Returns [`LoadBackendsError::PathNotUtf8`] when `BACKENDS_DIR` cannot be converted to UTF-8 -/// and [`LoadBackendsError::PathNullByte`] when it contains an interior null byte. -pub fn load_backends() -> Result<(), LoadBackendsError> { - let Some(dir) = BACKENDS_DIR else { - return Ok(()); - }; - - load_backends_from_path(Path::new(dir)) -} - -#[cfg(test)] -mod tests { - use super::load_backends; - - #[test] - fn load_backends_does_not_error_with_default_backends_dir() { - let result = load_backends(); - - assert!(result.is_ok()); - } -} diff --git a/llama-cpp-bindings/src/load_backends_error.rs b/llama-cpp-bindings/src/load_backends_error.rs index 3f7c5e6a3..258f52c35 100644 --- a/llama-cpp-bindings/src/load_backends_error.rs +++ b/llama-cpp-bindings/src/load_backends_error.rs @@ -1,7 +1,7 @@ use std::ffi::NulError; use std::path::PathBuf; -#[derive(Debug, thiserror::Error)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub enum LoadBackendsError { #[error("backend directory path is not valid UTF-8: {0}")] PathNotUtf8(PathBuf), diff --git a/llama-cpp-bindings/src/load_backends_from_path.rs b/llama-cpp-bindings/src/load_backends_from_path.rs index 2ce89ea1c..2658b627b 100644 --- a/llama-cpp-bindings/src/load_backends_from_path.rs +++ b/llama-cpp-bindings/src/load_backends_from_path.rs @@ -22,39 +22,46 @@ pub fn load_backends_from_path(path: &Path) -> Result<(), LoadBackendsError> { #[cfg(test)] mod tests { + use std::ffi::CString; + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + use std::path::PathBuf; + + use std::env; + use super::load_backends_from_path; use crate::load_backends_error::LoadBackendsError; - use std::path::PathBuf; #[test] fn load_backends_from_path_returns_path_null_byte_for_embedded_null() { - use std::ffi::CString; - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let path = PathBuf::from(OsStr::from_bytes(b"/tmp/foo\0bar")); - let err = load_backends_from_path(&path).unwrap_err(); - let representative = - LoadBackendsError::PathNullByte(CString::new(b"a\0b".to_vec()).unwrap_err()); assert_eq!( - std::mem::discriminant(&err), - std::mem::discriminant(&representative) + load_backends_from_path(&path), + Err(LoadBackendsError::PathNullByte( + CString::new("/tmp/foo\0bar").unwrap_err() + )) ); } #[test] fn load_backends_from_path_returns_path_not_utf8_for_invalid_utf8() { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let path = PathBuf::from(OsStr::from_bytes(b"/tmp/\xff\xfe")); - let err = load_backends_from_path(&path).unwrap_err(); - let representative = LoadBackendsError::PathNotUtf8(PathBuf::new()); assert_eq!( - std::mem::discriminant(&err), - std::mem::discriminant(&representative) + load_backends_from_path(&path), + Err(LoadBackendsError::PathNotUtf8(path.clone())) + ); + } + + #[test] + fn a_directory_without_backend_libraries_loads_no_backend() { + let registered_before = unsafe { llama_cpp_bindings_sys::ggml_backend_reg_count() }; + + assert_eq!(load_backends_from_path(&env::temp_dir()), Ok(())); + assert_eq!( + unsafe { llama_cpp_bindings_sys::ggml_backend_reg_count() }, + registered_before ); } } diff --git a/llama-cpp-bindings/src/log_options.rs b/llama-cpp-bindings/src/log_options.rs index 6192d0fff..f46b0f2fb 100644 --- a/llama-cpp-bindings/src/log_options.rs +++ b/llama-cpp-bindings/src/log_options.rs @@ -22,10 +22,10 @@ impl LogOptions { #[cfg(test)] mod tests { - use super::LogOptions; + use crate::log_options::LogOptions; #[test] - fn builder_chain_sets_both_flags() { + fn the_builder_disables_logs_and_demotes_info_to_debug() { let options = LogOptions::default() .with_logs_enabled(false) .with_demote_info_to_debug(true); diff --git a/llama-cpp-bindings/src/marker_role.rs b/llama-cpp-bindings/src/marker_role.rs deleted file mode 100644 index 665a4fba5..000000000 --- a/llama-cpp-bindings/src/marker_role.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::sampled_token_section::SampledTokenSection; - -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -/// A semantic transition performed when a streaming marker is consumed. -pub enum MarkerRole { - ReasoningOpen, - ReasoningClose, - ToolCallOpen, - ToolCallClose, -} - -impl MarkerRole { - #[must_use] - pub const fn opened_section(self) -> Option { - match self { - Self::ReasoningOpen => Some(SampledTokenSection::Reasoning), - Self::ToolCallOpen => Some(SampledTokenSection::ToolCall), - Self::ReasoningClose | Self::ToolCallClose => None, - } - } - - #[must_use] - pub const fn closed_section(self) -> Option { - match self { - Self::ReasoningClose => Some(SampledTokenSection::Reasoning), - Self::ToolCallClose => Some(SampledTokenSection::ToolCall), - Self::ReasoningOpen | Self::ToolCallOpen => None, - } - } -} - -#[cfg(test)] -mod tests { - use super::MarkerRole; - use crate::sampled_token_section::SampledTokenSection; - - struct RoleSection { - role: MarkerRole, - section: SampledTokenSection, - } - - #[test] - fn an_opening_role_opens_its_section_and_closes_nothing() { - let cases = [ - RoleSection { - role: MarkerRole::ReasoningOpen, - section: SampledTokenSection::Reasoning, - }, - RoleSection { - role: MarkerRole::ToolCallOpen, - section: SampledTokenSection::ToolCall, - }, - ]; - - for RoleSection { role, section } in cases { - assert_eq!(role.opened_section(), Some(section)); - assert_eq!(role.closed_section(), None); - } - } - - #[test] - fn a_closing_role_closes_its_section_and_opens_nothing() { - let cases = [ - RoleSection { - role: MarkerRole::ReasoningClose, - section: SampledTokenSection::Reasoning, - }, - RoleSection { - role: MarkerRole::ToolCallClose, - section: SampledTokenSection::ToolCall, - }, - ]; - - for RoleSection { role, section } in cases { - assert_eq!(role.closed_section(), Some(section)); - assert_eq!(role.opened_section(), None); - } - } -} diff --git a/llama-cpp-bindings/src/marker_role_candidate.rs b/llama-cpp-bindings/src/marker_role_candidate.rs deleted file mode 100644 index 6fd437378..000000000 --- a/llama-cpp-bindings/src/marker_role_candidate.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::marker_role::MarkerRole; -use crate::token::LlamaToken; - -#[derive(Clone, Debug, Eq, PartialEq)] -/// A tokenized marker string together with the single role it was detected as. -/// -/// Candidates are merged by token sequence, so the same tokens detected under two -/// roles become one [`crate::streaming_marker::StreamingMarker`] carrying both. -pub struct MarkerRoleCandidate { - pub tokens: Vec, - pub role: MarkerRole, -} diff --git a/llama-cpp-bindings/src/max_devices.rs b/llama-cpp-bindings/src/max_devices.rs index 014eeaee1..1d2058bc3 100644 --- a/llama-cpp-bindings/src/max_devices.rs +++ b/llama-cpp-bindings/src/max_devices.rs @@ -5,10 +5,10 @@ pub fn max_devices() -> usize { #[cfg(test)] mod tests { - use super::max_devices; + use crate::max_devices::max_devices; #[test] - fn returns_at_least_one() { + fn at_least_one_device_is_supported() { let device_count = max_devices(); assert!(device_count >= 1); diff --git a/llama-cpp-bindings/src/mlock_supported.rs b/llama-cpp-bindings/src/mlock_supported.rs deleted file mode 100644 index 96bf1728b..000000000 --- a/llama-cpp-bindings/src/mlock_supported.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[must_use] -pub fn mlock_supported() -> bool { - unsafe { llama_cpp_bindings_sys::llama_supports_mlock() } -} - -#[cfg(test)] -mod tests { - use super::mlock_supported; - - #[test] - fn returns_bool_without_panic() { - let _supported: bool = mlock_supported(); - } -} diff --git a/llama-cpp-bindings/src/mmap_supported.rs b/llama-cpp-bindings/src/mmap_supported.rs deleted file mode 100644 index 47ccbfe7b..000000000 --- a/llama-cpp-bindings/src/mmap_supported.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[must_use] -pub fn mmap_supported() -> bool { - unsafe { llama_cpp_bindings_sys::llama_supports_mmap() } -} - -#[cfg(test)] -mod tests { - use super::mmap_supported; - - #[test] - fn returns_bool_without_panic() { - let _supported: bool = mmap_supported(); - } -} diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 04105da10..e2309019f 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -10,71 +10,76 @@ pub mod llama_split_mode_parse_error; pub mod lora_adapter_scale; pub mod params; pub mod parse_special_tokens; +mod piece_write; pub mod rope_type; pub mod split_mode; pub mod tokenizer_input; pub mod vocab_type; pub mod vocab_type_from_int_error; +pub mod vocabulary_token; +use std::cmp; +use std::ffi::c_int; use std::ffi::{CStr, CString, c_char}; +use std::fmt; +use std::fmt::Debug; +use std::fmt::Formatter; use std::num::NonZeroU16; -use std::os::raw::c_int; +use std::num::TryFromIntError; use std::path::Path; use std::ptr; use std::ptr::NonNull; use std::sync::Arc; +use encoding_rs::CoderResult; use once_cell::sync::OnceCell; +use params::LlamaModelParams; use toktrie::ApproximateTokEnv; use toktrie::INVALID_TOKEN; use toktrie::TokRxInfo; use toktrie::TokTrie; -use llama_cpp_bindings_types::ReasoningMarkers; -use llama_cpp_bindings_types::ToolCallMarkers; - -use crate::bare_json_tool_calls::BareJsonToolCalls; -use crate::chat_template_tool_calls; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + +use crate::chat_templates::ChatTemplates; +use crate::error::chat_template_error::ChatTemplateError; +use crate::error::chat_templates_error::ChatTemplatesError; +use crate::error::llama_lora_adapter_init_error::LlamaLoraAdapterInitError; +use crate::error::llama_model_load_error::LlamaModelLoadError; +use crate::error::meta_val_error::MetaValError; +use crate::error::string_to_token_error::StringToTokenError; +use crate::error::token_attr_error::TokenAttrError; +use crate::error::token_outside_vocabulary::TokenOutsideVocabulary; +use crate::error::token_to_string_error::TokenToStringError; +use crate::error::unknown_rope_type::UnknownRopeType; use crate::llama_backend::LlamaBackend; use crate::llama_token_attrs::LlamaTokenAttrs; -use crate::marker_role::MarkerRole; -use crate::marker_role_candidate::MarkerRoleCandidate; +use crate::model::add_bos::AddBos; +use crate::model::llama_chat_template::LlamaChatTemplate; +use crate::model::llama_lora_adapter::LlamaLoraAdapter; +use crate::model::parse_special_tokens::ParseSpecialTokens; +use crate::model::piece_write::PieceWrite; +use crate::model::rope_type::RopeType; use crate::model::tokenizer_input::TokenizerInput; -use crate::resolved_tool_call_markers::ResolvedToolCallMarkers; +use crate::model::vocab_type::VocabType; +use crate::model::vocab_type_from_int_error::VocabTypeFromIntError; +use crate::model::vocabulary_token::VocabularyToken; use crate::sampled_token::SampledToken; -use crate::sampled_token_classifier::SampledTokenClassifier; -use crate::streaming_markers::StreamingMarkers; -use crate::synthetic_tool_call_renders::SyntheticToolCallRenders; use crate::token::LlamaToken; -use crate::{ - ApplyChatTemplateError, ChatTemplateError, LlamaLoraAdapterInitError, LlamaModelLoadError, - MarkerDetectionError, MetaValError, ParseChatMessageError, StringToTokenError, TokenAttrError, - TokenOutsideVocabulary, TokenToStringError, -}; - -pub use add_bos::AddBos; -pub use llama_chat_message::LlamaChatMessage; -pub use llama_chat_template::LlamaChatTemplate; -pub use llama_lazy_mode::LlamaLazyMode; -pub use llama_lazy_mode_parse_error::LlamaLazyModeParseError; -pub use llama_load_mode::LlamaLoadMode; -pub use llama_load_mode_parse_error::LlamaLoadModeParseError; -pub use llama_lora_adapter::LlamaLoraAdapter; -pub use parse_special_tokens::ParseSpecialTokens; -pub use rope_type::RopeType; -pub use vocab_type::VocabType; -pub use vocab_type_from_int_error::VocabTypeFromIntError; - -use llama_cpp_ffi_status::read_and_free_cpp_string; -use params::LlamaModelParams; + +const PIECE_PROBE_LEN: c_int = 32; + +const PIECE_PROBE_SIZE: usize = PIECE_PROBE_LEN.unsigned_abs() as usize; fn validate_string_length_for_tokenizer(length: usize) -> Result { Ok(c_int::try_from(length)?) } fn cstring_with_validated_len(text: &str) -> Result { + let length = validate_string_length_for_tokenizer(text.len())?; let c_string = CString::new(text)?; - let length = validate_string_length_for_tokenizer(c_string.as_bytes().len())?; Ok(TokenizerInput { text: c_string, @@ -92,84 +97,33 @@ const fn present_special_token( } } -pub struct LlamaModel { - pub model: NonNull, - tok_env: OnceCell>, - chat_parser: OnceCell, - reasoning_markers: OnceCell>, - streaming_markers: OnceCell>, -} - -#[derive(Debug)] -struct ChatParserHandle { - parser: NonNull, -} - -/// # Safety -/// -/// The handle is an opaque pointer to a heap-allocated parser owned by the model; it is -/// created once, never mutated afterwards, and freed exactly once on drop. The owning -/// `LlamaModel` is already `Send + Sync`, so the handle shares that guarantee. -unsafe impl Send for ChatParserHandle {} +fn decode_piece(decoder: &mut encoding_rs::Decoder, bytes: &[u8]) -> String { + let mut piece = String::with_capacity(bytes.len()); + let mut undecoded = bytes; -unsafe impl Sync for ChatParserHandle {} + loop { + let (result, read, _had_replacements) = + decoder.decode_to_string(undecoded, &mut piece, false); -/// # Safety -/// -/// `out_error` must be the pointer populated by the preceding -/// `llama_rs_chat_parser_free` call, or null. The destructor-threw arm reads and -/// frees it. -unsafe fn chat_parser_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_parser_free_status, - out_error: *mut c_char, -) -> Result<(), ParseChatMessageError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_chat_parser_free", - "reported a thrown C++ exception without an error message", - ) - }?; + undecoded = undecoded.split_at(read).1; - Err(ParseChatMessageError::DestructorFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_parser_free", - code: i64::from(other), + match result { + CoderResult::InputEmpty => return piece, + CoderResult::OutputFull => piece.reserve(char::MAX_LEN_UTF8), } - .into()), } } -impl Drop for ChatParserHandle { - fn drop(&mut self) { - let mut out_error: *mut c_char = ptr::null_mut(); - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_parser_free( - self.parser.as_ptr(), - &raw mut out_error, - ) - }; - - if let Err(destructor_failure) = - unsafe { chat_parser_free_status_to_result(status, out_error) } - { - log::error!("{destructor_failure}"); - } - } +pub struct LlamaModel { + pub model: NonNull, + vocab: *const llama_cpp_bindings_sys::llama_vocab, + n_vocab: i32, + tok_env: OnceCell>, + chat_templates: OnceCell, } -impl std::fmt::Debug for LlamaModel { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl Debug for LlamaModel { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("LlamaModel") .field("model", &self.model) .finish_non_exhaustive() @@ -194,17 +148,19 @@ unsafe fn load_model_from_file_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_OK => { let model = NonNull::new(out_model).ok_or_else(|| { - LlamaModelLoadError::from(crate::FfiContractError { + LlamaModelLoadError::from(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "success status contained a null model", }) })?; + let vocab = unsafe { llama_cpp_bindings_sys::llama_model_get_vocab(model.as_ptr()) }; + Ok(LlamaModel { model, + vocab, + n_vocab: unsafe { llama_cpp_bindings_sys::llama_vocab_n_tokens(vocab) }, tok_env: OnceCell::new(), - chat_parser: OnceCell::new(), - reasoning_markers: OnceCell::new(), - streaming_markers: OnceCell::new(), + chat_templates: OnceCell::new(), }) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_LLAMA_CPP_RETURNED_NULL => { @@ -231,27 +187,27 @@ unsafe fn load_model_from_file_status_to_result( Err(LlamaModelLoadError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_PATH_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null path argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_MODEL_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_load_model_from_file", code: i64::from(other), } @@ -259,208 +215,43 @@ unsafe fn load_model_from_file_status_to_result( } } -/// # Safety -/// -/// `out_error` must reference the pointer populated by the preceding -/// `llama_rs_chat_parser_create` call (or null); it is read, freed, and nulled only in the -/// CXX-exception arm. `parser` must be the pointer populated by the same call. -unsafe fn chat_parser_create_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_chat_parser_create_status, - parser: *mut llama_cpp_bindings_sys::llama_rs_chat_parser, - out_error: *mut *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_OK => NonNull::new(parser) - .map(|parser| ChatParserHandle { parser }) - .ok_or_else(|| { - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "success status contained a null parser handle", - } - .into() - }), - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_CHAT_TEMPLATE => { - Err(ParseChatMessageError::NoChatTemplate) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_VOCAB => { - Err(ParseChatMessageError::NoVocab) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ParseChatMessageError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ParseChatMessageError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - *out_error, - "llama_rs_chat_parser_create", - "reported a thrown C++ exception without an error message", - ) - }?; - unsafe { *out_error = ptr::null_mut() }; - Err(ParseChatMessageError::ParserCreationFailed { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_MODEL_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null model argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_PARSER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null out_parser argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_chat_parser_create", - code: i64::from(other), - } - .into()), - } -} - -/// # Safety -/// -/// `out_string` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_apply_chat_template` call (or null). The success arm reads and frees -/// `out_string`; the CXX-exception arm reads and frees `out_error`. -unsafe fn apply_chat_template_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_apply_chat_template_status, - out_string: *mut c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_OK => Ok(unsafe { - read_and_free_cpp_string( - out_string, - "llama_rs_apply_chat_template", - "success status contained a null rendered string", - ) - }?), - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_MODEL_HAS_NO_VOCAB => { - Err(ApplyChatTemplateError::NoVocab) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_TEMPLATE_APPLICATION_FAILED => { - Err(ApplyChatTemplateError::TemplateApplicationFailed) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED => { - Err(ApplyChatTemplateError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(ApplyChatTemplateError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_apply_chat_template", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(ApplyChatTemplateError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MODEL_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null model argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_TEMPLATE_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null template argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MESSAGES_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null messages argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_STRING_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null out_string argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_apply_chat_template", - code: i64::from(other), - } - .into()), - } -} - impl LlamaModel { fn require_vocabulary_token( &self, token: LlamaToken, ) -> Result { - let n_vocab = self.n_vocab(); - - if (0..n_vocab).contains(&token.0) { - Ok(token) - } else { - Err(TokenOutsideVocabulary { token, n_vocab }) - } + TokenOutsideVocabulary::check(token, self.n_vocab()) } #[must_use] - pub fn vocab_ptr(&self) -> *const llama_cpp_bindings_sys::llama_vocab { - unsafe { llama_cpp_bindings_sys::llama_model_get_vocab(self.model.as_ptr()) } + pub const fn vocab_ptr(&self) -> *const llama_cpp_bindings_sys::llama_vocab { + self.vocab } /// # Errors /// /// Returns an error if the value returned by llama.cpp does not fit into a `u32`. - pub fn n_ctx_train(&self) -> Result { + pub fn n_ctx_train(&self) -> Result { let n_ctx_train = unsafe { llama_cpp_bindings_sys::llama_model_n_ctx_train(self.model.as_ptr()) }; u32::try_from(n_ctx_train) } + /// Every token of the vocabulary, in id order, with the text it decodes to on its own. pub fn tokens( &self, decode_special: bool, - ) -> impl Iterator)> + '_ { - (0..self.n_vocab()) - .map(LlamaToken::new) - .map(move |llama_token| { - let mut decoder = encoding_rs::UTF_8.new_decoder(); - ( - llama_token, - self.token_to_piece( - &SampledToken::Content(llama_token), - &mut decoder, - decode_special, - None, - ), - ) - }) + ) -> impl Iterator> + '_ { + (0..self.n_vocab()).map(LlamaToken::new).map(move |token| { + self.token_to_piece( + &SampledToken::Content(token), + &mut encoding_rs::UTF_8.new_decoder(), + decode_special, + None, + ) + .map(|piece| VocabularyToken { token, piece }) + }) } #[must_use] @@ -524,7 +315,7 @@ impl LlamaModel { ParseSpecialTokens::Never => false, }; - let tokens_estimation = std::cmp::max(8, (str.len() / 2) + usize::from(add_bos)); + let tokens_estimation = cmp::max(8, (str.len() / 2) + usize::from(add_bos)); let TokenizerInput { text: c_string, length: c_string_len, @@ -547,14 +338,14 @@ impl LlamaModel { /// # Errors /// /// Returns [`TokenAttrError::TokenOutsideVocabulary`] when the token is not part of this - /// model's vocabulary, or [`TokenAttrError::UnknownAttributes`] when the token type is not - /// known to this library. + /// model's vocabulary, or [`TokenAttrError::UnknownAttributes`] when llama.cpp reports + /// attributes this library does not know. pub fn token_attr(&self, token: LlamaToken) -> Result { let LlamaToken(id) = self.require_vocabulary_token(token)?; - let token_type = + let token_attr = unsafe { llama_cpp_bindings_sys::llama_vocab_get_attr(self.vocab_ptr(), id) }; - LlamaTokenAttrs::try_from(token_type).map_err(TokenAttrError::UnknownAttributes) + LlamaTokenAttrs::try_from(token_attr).map_err(TokenAttrError::UnknownAttributes) } /// # Errors @@ -573,20 +364,7 @@ impl LlamaModel { | SampledToken::Reasoning(inner) | SampledToken::ToolCall(inner) | SampledToken::Undeterminable(inner)) = *token; - let bytes = match self.token_to_piece_bytes(inner, 8, special, lstrip) { - Err(TokenToStringError::InsufficientBufferSpace(required_size)) => { - let buffer_size: usize = (-required_size).try_into()?; - - self.token_to_piece_bytes(inner, buffer_size, special, lstrip) - } - other => other, - }?; - - let mut output_piece = String::with_capacity(bytes.len()); - let (_result, _decoded_size, _had_replacements) = - decoder.decode_to_string(&bytes, &mut output_piece, false); - - Ok(output_piece) + self.read_piece(inner, special, lstrip, |piece| decode_piece(decoder, piece)) } /// # Errors @@ -603,37 +381,61 @@ impl LlamaModel { lstrip: Option, ) -> Result, TokenToStringError> { let token = self.require_vocabulary_token(token)?; + let buffer_len = c_int::try_from(buffer_size)?; let mut buffer: Vec = vec![0u8; buffer_size]; - let buffer_len = c_int::try_from(buffer.len())?; - let lstrip = lstrip.map_or(0, |strip_count| i32::from(strip_count.get())); - let size = unsafe { - llama_cpp_bindings_sys::llama_token_to_piece( - self.vocab_ptr(), + match self.write_piece(token, &mut buffer, buffer_len, special, lstrip) { + PieceWrite::Written(written) => { + buffer.truncate(written); + + Ok(buffer) + } + PieceWrite::NeedsBuffer(negated_size) => { + Err(TokenToStringError::InsufficientBufferSpace(negated_size)) + } + } + } + + fn write_piece( + &self, + token: LlamaToken, + buffer: &mut [u8], + buffer_len: c_int, + special: bool, + lstrip: Option, + ) -> PieceWrite { + PieceWrite::from(unsafe { + llama_cpp_bindings_sys::llama_rs_token_to_piece( + self.vocab, token.0, buffer.as_mut_ptr().cast::(), buffer_len, - lstrip, + lstrip.map_or(0, |strip_count| i32::from(strip_count.get())), special, ) - }; + }) + } - match size { - 0 => Err(TokenToStringError::UnknownTokenType), - error_code if error_code.is_negative() => { - Err(TokenToStringError::InsufficientBufferSpace(error_code)) - } - size => { - let written = usize::try_from(size)?; - buffer.truncate(written); + fn read_piece( + &self, + token: LlamaToken, + special: bool, + lstrip: Option, + read: impl FnOnce(&[u8]) -> TOutput, + ) -> Result { + let token = self.require_vocabulary_token(token)?; + let mut probe = [0_u8; PIECE_PROBE_SIZE]; - Ok(buffer) - } + match self.write_piece(token, &mut probe, PIECE_PROBE_LEN, special, lstrip) { + PieceWrite::Written(written) => Ok(read(probe.split_at(written).0)), + PieceWrite::NeedsBuffer(negated_size) => self + .token_to_piece_bytes(token, negated_size.unsigned_abs() as usize, special, lstrip) + .map(|piece| read(&piece)), } } #[must_use] - pub fn n_vocab(&self) -> i32 { - unsafe { llama_cpp_bindings_sys::llama_vocab_n_tokens(self.vocab_ptr()) } + pub const fn n_vocab(&self) -> i32 { + self.n_vocab } /// # Errors @@ -678,21 +480,21 @@ impl LlamaModel { /// # Errors /// /// Returns an error if the layer count returned by llama.cpp does not fit into a `u32`. - pub fn n_layer(&self) -> Result { + pub fn n_layer(&self) -> Result { u32::try_from(unsafe { llama_cpp_bindings_sys::llama_model_n_layer(self.model.as_ptr()) }) } /// # Errors /// /// Returns an error if the head count returned by llama.cpp does not fit into a `u32`. - pub fn n_head(&self) -> Result { + pub fn n_head(&self) -> Result { u32::try_from(unsafe { llama_cpp_bindings_sys::llama_model_n_head(self.model.as_ptr()) }) } /// # Errors /// /// Returns an error if the KV head count returned by llama.cpp does not fit into a `u32`. - pub fn n_head_kv(&self) -> Result { + pub fn n_head_kv(&self) -> Result { u32::try_from(unsafe { llama_cpp_bindings_sys::llama_model_n_head_kv(self.model.as_ptr()) }) } @@ -701,6 +503,11 @@ impl LlamaModel { unsafe { llama_cpp_bindings_sys::llama_model_is_hybrid(self.model.as_ptr()) } } + #[must_use] + pub fn has_encoder(&self) -> bool { + unsafe { llama_cpp_bindings_sys::llama_model_has_encoder(self.model.as_ptr()) } + } + /// # Errors /// Returns an error if the key is not found or the value is not valid UTF-8. pub fn meta_val_str(&self, key: &str) -> Result { @@ -757,30 +564,25 @@ impl LlamaModel { ) } - #[must_use] - pub fn rope_type(&self) -> Option { - let raw = unsafe { llama_cpp_bindings_sys::llama_model_rope_type(self.model.as_ptr()) }; - - RopeType::from_raw(raw) + /// # Errors + /// + /// Returns [`UnknownRopeType`] when llama.cpp reports a rope type it does not define. + pub fn rope_type(&self) -> Result { + RopeType::try_from(unsafe { + llama_cpp_bindings_sys::llama_rs_model_rope_type(self.model.as_ptr()) + }) } /// # Errors /// - /// * If the model has no chat template by that name - /// - /// # Panics - /// - /// Panics if the C-returned chat template string contains interior null bytes - /// (should never happen with valid model data). + /// Returns [`ChatTemplateError::NullError`] when `name` contains a NUL byte, or + /// [`ChatTemplateError::MissingTemplate`] when the model has no chat template by that name. pub fn chat_template( &self, name: Option<&str>, ) -> Result { - let name_cstr = name.map(CString::new); - let name_ptr = match name_cstr { - Some(Ok(name)) => name.as_ptr(), - _ => ptr::null(), - }; + let name_cstr = name.map(CString::new).transpose()?; + let name_ptr = name_cstr.as_ref().map_or(ptr::null(), |name| name.as_ptr()); let result = unsafe { llama_cpp_bindings_sys::llama_model_chat_template(self.model.as_ptr(), name_ptr) }; @@ -811,12 +613,11 @@ impl LlamaModel { let path_str = path .to_str() .ok_or_else(|| LlamaModelLoadError::PathToStrError(path.to_path_buf()))?; + let cstr = CString::new(path_str)?; if !path.exists() { return Err(LlamaModelLoadError::FileNotFound(path.to_path_buf())); } - - let cstr = CString::new(path_str)?; let mut out_model: *mut llama_cpp_bindings_sys::llama_model = ptr::null_mut(); let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { @@ -842,255 +643,22 @@ impl LlamaModel { let path_str = path .to_str() .ok_or_else(|| LlamaLoraAdapterInitError::PathToStrError(path.to_path_buf()))?; + let cstr = CString::new(path_str)?; if !path.exists() { return Err(LlamaLoraAdapterInitError::FileNotFound(path.to_path_buf())); } - - let cstr = CString::new(path_str)?; initialize_lora_adapter(|| unsafe { llama_cpp_bindings_sys::llama_adapter_lora_init(self.model.as_ptr(), cstr.as_ptr()) }) } - - /// # Errors - /// Returns [`ApplyChatTemplateError`] if the model has no vocab, the template - /// renders an empty prompt or cannot be rendered, or the renderer throws. - pub fn apply_chat_template( - &self, - tmpl: &LlamaChatTemplate, - chat: &[LlamaChatMessage], - add_ass: bool, - enable_thinking: bool, - ) -> Result { - let roles: Vec<*const c_char> = chat - .iter() - .map(|chat_message| chat_message.role.as_ptr()) - .collect(); - let contents: Vec<*const c_char> = chat - .iter() - .map(|chat_message| chat_message.content.as_ptr()) - .collect(); - - let mut out_string: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_apply_chat_template( - self.model.as_ptr(), - tmpl.0.as_ptr(), - roles.as_ptr(), - contents.as_ptr(), - chat.len(), - i32::from(add_ass), - i32::from(enable_thinking), - &raw mut out_string, - &raw mut out_error, - ) - }; - - unsafe { apply_chat_template_status_to_result(status, out_string, out_error) } - } - - /// # Errors - /// Returns [`MarkerDetectionError`] when streaming-marker detection fails. - /// The classifier is never constructed in a degraded "blind" state — a - /// detection failure is surfaced to the caller instead of silently ignored. - pub fn sampled_token_classifier( - &self, - bare_json_tool_calls: BareJsonToolCalls, - ) -> Result, MarkerDetectionError> { - self.streaming_markers() - .map(|markers| SampledTokenClassifier::new(self, markers, bare_json_tool_calls)) - } - - /// # Errors - /// Returns [`MarkerDetectionError`] when any underlying FFI call fails. - pub fn streaming_markers(&self) -> Result, MarkerDetectionError> { - self.streaming_markers - .get_or_try_init(|| self.detect_streaming_markers().map(Arc::new)) - .map(Arc::clone) - } - - fn detect_streaming_markers(&self) -> Result { - let reasoning_markers = self.reasoning_markers()?; - - let tool_call_haystack = invoke_compute_tool_call_haystack(self.model.as_ptr())?; - - let autoparser_pair = tool_call_haystack.as_deref().and_then( - crate::extract_tool_call_markers_from_haystack::extract_tool_call_markers_from_haystack, - ); - - let (autoparser_open, autoparser_close) = match autoparser_pair { - Some(crate::tool_call_marker_pair::ToolCallMarkerPair { open, close }) => { - (Some(open), Some(close)) - } - None => (None, None), - }; - - let resolved_tool_call_markers = - self.resolve_tool_call_marker_strings(autoparser_open, autoparser_close)?; - - let mut candidates = Vec::new(); - if let Some(markers) = reasoning_markers { - for marker in &markers.closes { - if let Some(tokens) = self.tokenize_marker(Some(marker))? { - candidates.push(MarkerRoleCandidate { - tokens, - role: MarkerRole::ReasoningClose, - }); - } - } - } - - let reasoning_open = reasoning_markers.map(|markers| markers.open.as_str()); - if let Some(tokens) = self.tokenize_marker(reasoning_open)? { - candidates.push(MarkerRoleCandidate { - tokens, - role: MarkerRole::ReasoningOpen, - }); - } - if let Some(tokens) = self.tokenize_marker(resolved_tool_call_markers.open.as_deref())? { - candidates.push(MarkerRoleCandidate { - tokens, - role: MarkerRole::ToolCallOpen, - }); - } - if let Some(tokens) = self.tokenize_marker(resolved_tool_call_markers.close.as_deref())? { - candidates.push(MarkerRoleCandidate { - tokens, - role: MarkerRole::ToolCallClose, - }); - } - - StreamingMarkers::from_candidates(candidates) - } - - fn resolve_tool_call_marker_strings( - &self, - autoparser_open: Option, - autoparser_close: Option, - ) -> Result { - if autoparser_open - .as_deref() - .is_some_and(|raw| !raw.trim().is_empty()) - { - return Ok(ResolvedToolCallMarkers { - open: autoparser_open, - close: autoparser_close, - }); - } - let Some(markers) = self.tool_call_markers()? else { - return Ok(ResolvedToolCallMarkers { - open: autoparser_open, - close: autoparser_close, - }); - }; - let close = if markers.close.is_empty() { - None - } else { - Some(markers.close) - }; - Ok(ResolvedToolCallMarkers { - open: Some(markers.open), - close, - }) - } - - /// # Errors - /// Returns [`MarkerDetectionError`] when the underlying FFI call fails. - pub fn reasoning_markers(&self) -> Result, MarkerDetectionError> { - self.reasoning_markers - .get_or_try_init(|| invoke_detect_reasoning_markers(self.model.as_ptr())) - .map(Option::as_ref) - } - - /// # Errors - /// Returns [`MarkerDetectionError::ToolCallTemplateNotUtf8`] when the model - /// has a chat template that is not valid UTF-8. A model with no chat - /// template legitimately yields `Ok(None)`. - pub fn tool_call_markers(&self) -> Result, MarkerDetectionError> { - let template = match self.chat_template(None) { - Ok(template) => template, - Err(ChatTemplateError::MissingTemplate) => return Ok(None), - Err(other) => return Err(MarkerDetectionError::ChatTemplateUnavailable(other)), - }; - let template_str = template.to_str()?; - - Ok(chat_template_tool_calls::detect(template_str)) - } - - /// # Errors - /// Returns [`StringToTokenError`] when a present, non-empty marker string - /// fails to tokenise. - fn tokenize_marker( - &self, - marker: Option<&str>, - ) -> Result>, StringToTokenError> { - let Some(marker) = marker else { - return Ok(None); - }; - let marker = marker.trim(); - if marker.is_empty() { - return Ok(None); - } - let tokens = self.str_to_token(marker, AddBos::Never, ParseSpecialTokens::Always)?; - if tokens.is_empty() { - Ok(None) - } else { - Ok(Some(tokens)) - } - } - - /// Returns the model's chat parser, analysing the chat template on first use. The - /// pointer stays valid for as long as the model lives. - /// - /// # Errors - /// - /// Returns [`ParseChatMessageError`] when the model has no chat template or vocab, or - /// the parser cannot be constructed. - pub fn chat_parser_ptr( - &self, - ) -> Result<*mut llama_cpp_bindings_sys::llama_rs_chat_parser, ParseChatMessageError> { - self.chat_parser().map(|handle| handle.parser.as_ptr()) - } - - fn chat_parser(&self) -> Result<&ChatParserHandle, ParseChatMessageError> { - self.chat_parser - .get_or_try_init(|| self.create_chat_parser()) - } - - fn create_chat_parser(&self) -> Result { - let mut out_parser: *mut llama_cpp_bindings_sys::llama_rs_chat_parser = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_chat_parser_create( - self.model.as_ptr(), - &raw mut out_parser, - &raw mut out_error, - ) - }; - - unsafe { chat_parser_create_status_to_result(status, out_parser, &raw mut out_error) } - } - - /// # Errors - /// - /// Returns [`MarkerDetectionError`] when the C++ analyzer throws or the FFI - /// returns a non-OK status. - pub fn diagnose_tool_call_synthetic_renders( - &self, - ) -> Result { - invoke_diagnose_tool_call_synthetic_renders(self.model.as_ptr()) - } } fn initialize_lora_adapter<'model>( initialize: impl FnOnce() -> *mut llama_cpp_bindings_sys::llama_adapter_lora, ) -> Result, LlamaLoraAdapterInitError> { let adapter = NonNull::new(initialize()).ok_or(LlamaLoraAdapterInitError::Unloadable)?; - Ok(LlamaLoraAdapter::new(adapter)) + Ok(unsafe { LlamaLoraAdapter::new(adapter) }) } impl LlamaModel { @@ -1104,24 +672,18 @@ impl LlamaModel { .get_or_try_init(|| build_approximate_tok_env(self)) .map(Arc::clone) } -} -const TOK_ENV_PIECE_PROBE_SIZE: usize = 32; - -fn token_piece_bytes_for_tok_env( - model: &LlamaModel, - token: LlamaToken, - special: bool, -) -> Result, TokenToStringError> { - match model.token_to_piece_bytes(token, TOK_ENV_PIECE_PROBE_SIZE, special, None) { - Ok(bytes) => Ok(bytes), - Err(TokenToStringError::UnknownTokenType) => Ok(Vec::new()), - Err(TokenToStringError::InsufficientBufferSpace(required)) => { - model.token_to_piece_bytes(token, required.unsigned_abs() as usize, special, None) - } - Err(other) => Err(other), - } -} + /// Returns the model's chat templates, preparing them on first use. + /// + /// # Errors + /// + /// Returns [`ChatTemplatesError`] when the model has no chat template or llama.cpp cannot + /// prepare it. + pub fn chat_templates(&self) -> Result<&ChatTemplates, ChatTemplatesError> { + self.chat_templates + .get_or_try_init(|| ChatTemplates::new(self)) + } +} fn build_approximate_tok_env( model: &LlamaModel, @@ -1141,9 +703,9 @@ fn build_approximate_tok_env( for token_id in 0..n_vocab.cast_signed() { let token = LlamaToken(token_id); - let bytes = token_piece_bytes_for_tok_env(model, token, false)?; + let bytes = model.read_piece(token, false, None, <[u8]>::to_vec)?; if bytes.is_empty() { - let special_bytes = token_piece_bytes_for_tok_env(model, token, true)?; + let special_bytes = model.read_piece(token, true, None, <[u8]>::to_vec)?; if special_bytes.is_empty() { words.push(vec![]); } else { @@ -1161,390 +723,6 @@ fn build_approximate_tok_env( Ok(Arc::new(ApproximateTokEnv::new(trie))) } -unsafe fn detect_reasoning_markers_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers_status, - out_markers: *const llama_cpp_bindings_sys::llama_rs_reasoning_markers, - out_error: *mut c_char, -) -> Result, MarkerDetectionError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_OK => unsafe { - read_reasoning_markers(out_markers) - }, - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_MODEL_ARG => { - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "model", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_MARKERS_ARG => { - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "out_markers", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG => { - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "out_error", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_CHAT_TEMPLATE => Ok(None), - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_MODEL_HAS_NO_VOCAB => { - Err(MarkerDetectionError::ModelHasNoVocab { - operation: "llama_rs_detect_reasoning_markers", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED => { - Err(MarkerDetectionError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MarkerDetectionError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_detect_reasoning_markers", "reported a thrown C++ exception without an error message") }?; - Err(MarkerDetectionError::ReasoningMarkerDetectionFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_detect_reasoning_markers", - code: i64::from(other), - } - .into()), - } -} - -unsafe fn read_reasoning_markers( - markers: *const llama_cpp_bindings_sys::llama_rs_reasoning_markers, -) -> Result, MarkerDetectionError> { - if markers.is_null() { - return Ok(None); - } - let open_pointer = unsafe { llama_cpp_bindings_sys::llama_rs_reasoning_markers_open(markers) }; - let open = read_optional_owned_cstr(open_pointer)?; - let close_count = - unsafe { llama_cpp_bindings_sys::llama_rs_reasoning_markers_close_count(markers) }; - let mut closes = Vec::with_capacity(close_count); - for index in 0..close_count { - let close_pointer = - unsafe { llama_cpp_bindings_sys::llama_rs_reasoning_markers_close_at(markers, index) }; - closes.push(read_optional_owned_cstr(close_pointer)?); - } - validate_reasoning_markers(open, closes).map(Some) -} - -fn validate_reasoning_markers( - open: Option, - closes: Vec>, -) -> Result { - let Some(open) = open else { - return Err(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_open", - detail: "non-null markers returned a null opening marker", - } - .into()); - }; - if open.is_empty() || closes.is_empty() { - return Err(crate::FfiContractError { - operation: "llama_rs_detect_reasoning_markers", - detail: "detected markers must contain an opening marker and a closing marker", - } - .into()); - } - let closes = closes - .into_iter() - .map(|close| { - let Some(close) = close else { - return Err(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_close_at", - detail: "a valid closing-marker index returned null", - } - .into()); - }; - if close.is_empty() { - return Err(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_close_at", - detail: "a detected closing marker was empty", - } - .into()); - } - Ok(close) - }) - .collect::, MarkerDetectionError>>()?; - Ok(ReasoningMarkers { open, closes }) -} - -const fn cxx_exception_owns_out_error( - parsed: &Result, -) -> bool { - matches!( - parsed, - Err(MarkerDetectionError::ReasoningMarkerDetectionFailed { .. } - | MarkerDetectionError::ToolCallHaystackComputationFailed { .. } - | MarkerDetectionError::ToolCallSyntheticRenderDiagnosisFailed { .. }) - ) -} - -/// # Safety -/// -/// `free_error` must be the pointer populated by the preceding -/// `llama_rs_reasoning_markers_free` call, or null. The destructor-threw arm -/// reads and frees it. -unsafe fn reasoning_markers_free_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_reasoning_markers_free_status, - free_error: *mut c_char, -) -> Result<(), MarkerDetectionError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_ERROR_STRING_ALLOCATION_FAILED => { - Err(MarkerDetectionError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MarkerDetectionError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - free_error, - "llama_rs_reasoning_markers_free", - "reported a thrown C++ exception without an error message", - ) - }?; - - Err(MarkerDetectionError::ReasoningMarkersFreeFailed { message }) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_reasoning_markers_free", - code: i64::from(other), - } - .into()), - } -} - -fn invoke_detect_reasoning_markers( - model: *const llama_cpp_bindings_sys::llama_model, -) -> Result, MarkerDetectionError> { - let mut out_markers: *mut llama_cpp_bindings_sys::llama_rs_reasoning_markers = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers( - model, - &raw mut out_markers, - &raw mut out_error, - ) - }; - - let parsed = - unsafe { detect_reasoning_markers_status_to_result(status, out_markers, out_error) }; - - let mut free_error: *mut c_char = ptr::null_mut(); - let free_status = unsafe { - llama_cpp_bindings_sys::llama_rs_reasoning_markers_free(out_markers, &raw mut free_error) - }; - let freed = unsafe { reasoning_markers_free_status_to_result(free_status, free_error) }; - - if !cxx_exception_owns_out_error(&parsed) { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - } - - match parsed { - Ok(markers) => freed.map(|()| markers), - Err(detection_failure) => { - if let Err(destructor_failure) = freed { - log::error!("{destructor_failure}"); - } - - Err(detection_failure) - } - } -} - -/// # Safety -/// -/// `out_haystack` and `out_error` must be the pointers populated by the preceding -/// `llama_rs_compute_tool_call_haystack` call (or null). `out_haystack` is read but not freed -/// here; `out_error` is freed only in the CXX-exception arm, mirroring the conditional cleanup -/// in the caller. -unsafe fn compute_tool_call_haystack_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_compute_tool_call_haystack_status, - out_haystack: *const c_char, - out_error: *mut c_char, -) -> Result, MarkerDetectionError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK => { - read_optional_owned_cstr(out_haystack) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_CHAT_TEMPLATE => Ok(None), - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_MODEL_HAS_NO_VOCAB => { - Err(MarkerDetectionError::ModelHasNoVocab { - operation: "llama_rs_compute_tool_call_haystack", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED => { - Err(MarkerDetectionError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MarkerDetectionError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_compute_tool_call_haystack", "reported a thrown C++ exception without an error message") }?; - Err(MarkerDetectionError::ToolCallHaystackComputationFailed { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_MODEL_ARG => Err(crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null model argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_HAYSTACK_ARG => Err(crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null out_haystack argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_ERROR_ARG => Err(crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null out_error argument", - } - .into()), - other => Err(crate::FfiStatusError { - operation: "llama_rs_compute_tool_call_haystack", - code: i64::from(other), - } - .into()), - } -} - -fn invoke_compute_tool_call_haystack( - model: *const llama_cpp_bindings_sys::llama_model, -) -> Result, MarkerDetectionError> { - let mut out_haystack: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_compute_tool_call_haystack( - model, - &raw mut out_haystack, - &raw mut out_error, - ) - }; - - let parsed = - unsafe { compute_tool_call_haystack_status_to_result(status, out_haystack, out_error) }; - - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_haystack) }; - if !cxx_exception_owns_out_error(&parsed) { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - } - - parsed -} - -/// # Safety -/// -/// `out_no_tools`, `out_with_tools`, and `out_error` must be the pointers populated by the -/// preceding `llama_rs_diagnose_tool_call_synthetic_renders` call (or null). The render -/// pointers are read but not freed here; `out_error` is freed only in the CXX-exception arm, -/// mirroring the cleanup in the caller. -unsafe fn diagnose_tool_call_synthetic_renders_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_diagnose_tool_call_synthetic_renders_status, - out_no_tools: *const c_char, - out_with_tools: *const c_char, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_OK => { - collect_synthetic_tool_call_renders(out_no_tools, out_with_tools) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_CHAT_TEMPLATE => { - Err(MarkerDetectionError::ModelHasNoChatTemplate { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_MODEL_HAS_NO_VOCAB => { - Err(MarkerDetectionError::ModelHasNoVocab { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - }) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED => { - Err(MarkerDetectionError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MarkerDetectionError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_diagnose_tool_call_synthetic_renders", "reported a thrown C++ exception without an error message") }?; - Err(MarkerDetectionError::ToolCallSyntheticRenderDiagnosisFailed { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_MODEL_ARG => Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null model argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_NO_TOOLS_ARG => Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_no_tools argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_WITH_TOOLS_ARG => Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_with_tools argument", - } - .into()), - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_ERROR_ARG => Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_error argument", - } - .into()), - other => Err(crate::FfiStatusError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - code: i64::from(other), - } - .into()), - } -} - -fn invoke_diagnose_tool_call_synthetic_renders( - model: *const llama_cpp_bindings_sys::llama_model, -) -> Result { - let mut out_no_tools: *mut c_char = ptr::null_mut(); - let mut out_with_tools: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_diagnose_tool_call_synthetic_renders( - model, - &raw mut out_no_tools, - &raw mut out_with_tools, - &raw mut out_error, - ) - }; - - let parsed = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - status, - out_no_tools, - out_with_tools, - out_error, - ) - }; - - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_no_tools) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_with_tools) }; - if !cxx_exception_owns_out_error(&parsed) { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - } - - parsed -} - -fn read_optional_owned_cstr(ptr: *const c_char) -> Result, MarkerDetectionError> { - if ptr.is_null() { - return Ok(None); - } - - let bytes = unsafe { CStr::from_ptr(ptr) }.to_bytes().to_vec(); - - Ok(Some(String::from_utf8(bytes)?)) -} - /// # Safety /// /// `out_error` must be the pointer populated by the preceding `llama_rs_tokenize` call (or @@ -1574,7 +752,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null vocab argument", } @@ -1582,7 +760,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null text argument", } @@ -1590,7 +768,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_RETURNED_COUNT_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_returned_count argument", } @@ -1598,7 +776,7 @@ unsafe fn tokenize_status_to_result( } llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_ERROR_ARG => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_tokenize", detail: "was given a null out_error argument", } @@ -1606,7 +784,7 @@ unsafe fn tokenize_status_to_result( } other => { unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - Err(crate::FfiStatusError { + Err(FfiStatusError { operation: "llama_rs_tokenize", code: i64::from(other), } @@ -1657,8 +835,8 @@ fn tokenize_into_buffer( c_int, ) -> Result, ) -> Result, StringToTokenError> { + let buffer_capacity = checked_token_buffer_capacity(estimated_capacity)?; let mut buffer: Vec = Vec::with_capacity(estimated_capacity); - let buffer_capacity = checked_token_buffer_capacity(buffer.capacity())?; let size = invoke( buffer @@ -1668,12 +846,14 @@ fn tokenize_into_buffer( )?; let size = if size.is_negative() { - buffer.reserve_exact(checked_token_count(-size)?); + let required_size = size.wrapping_neg(); + + buffer.reserve_exact(checked_token_count(required_size)?); invoke( buffer .as_mut_ptr() .cast::(), - -size, + required_size, )? } else { size @@ -1686,26 +866,6 @@ fn tokenize_into_buffer( Ok(buffer) } -fn collect_synthetic_tool_call_renders( - without_tools_ptr: *const c_char, - with_tools_ptr: *const c_char, -) -> Result { - let without_tools = - read_optional_owned_cstr(without_tools_ptr)?.ok_or(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "success status contained a null render without tools", - })?; - let with_tools = read_optional_owned_cstr(with_tools_ptr)?.ok_or(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "success status contained a null render with tools", - })?; - - Ok(SyntheticToolCallRenders { - without_tools, - with_tools, - }) -} - fn extract_meta_string( c_function: TCFunction, capacity: usize, @@ -1743,25 +903,32 @@ impl Drop for LlamaModel { #[cfg(test)] mod cstring_with_validated_len_tests { + use std::ffi::CString; + use std::str; + use super::cstring_with_validated_len; + use crate::error::string_to_token_error::StringToTokenError; #[test] - fn cstring_with_validated_len_null_byte_returns_error() { - let result = cstring_with_validated_len("null\0byte"); - - assert!(result.is_err()); + fn cstring_with_validated_len_reports_a_nul_byte_in_the_text() { + assert_eq!( + cstring_with_validated_len("null\0byte").err(), + CString::new("null\0byte") + .err() + .map(StringToTokenError::NulError) + ); } -} - -#[cfg(test)] -mod validate_string_length_for_tokenizer_tests { - use super::validate_string_length_for_tokenizer; #[test] - fn validate_string_length_overflow_returns_error() { - let result = validate_string_length_for_tokenizer(usize::MAX); + fn a_text_longer_than_a_c_int_is_refused_before_it_is_copied() { + let too_long = vec![0_u8; usize::try_from(i32::MAX).unwrap() + 1]; - assert!(result.is_err()); + assert_eq!( + cstring_with_validated_len(str::from_utf8(&too_long).unwrap()).err(), + i32::try_from(too_long.len()) + .err() + .map(StringToTokenError::CIntConversionError) + ); } } @@ -1769,11 +936,6 @@ mod validate_string_length_for_tokenizer_tests { mod checked_token_buffer_capacity_tests { use super::checked_token_buffer_capacity; - #[test] - fn checked_token_buffer_capacity_overflow_returns_error() { - assert!(checked_token_buffer_capacity(usize::MAX).is_err()); - } - #[test] fn checked_token_buffer_capacity_in_range_returns_value() { assert_eq!(checked_token_buffer_capacity(8), Ok(8)); @@ -1784,11 +946,6 @@ mod checked_token_buffer_capacity_tests { mod checked_token_count_tests { use super::checked_token_count; - #[test] - fn checked_token_count_negative_returns_error() { - assert!(checked_token_count(-1).is_err()); - } - #[test] fn checked_token_count_non_negative_returns_value() { assert_eq!(checked_token_count(5), Ok(5)); @@ -1798,7 +955,10 @@ mod checked_token_count_tests { #[cfg(test)] mod tokenize_into_buffer_tests { + use std::cell::Cell; + use super::tokenize_into_buffer; + use crate::error::string_to_token_error::StringToTokenError; #[test] fn tokenize_into_buffer_single_pass_sets_length() { @@ -1809,7 +969,7 @@ mod tokenize_into_buffer_tests { #[test] fn tokenize_into_buffer_grows_buffer_when_first_pass_reports_negative_size() { - let call_count = std::cell::Cell::new(0); + let call_count = Cell::new(0); let buffer = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { let count = call_count.get(); call_count.set(count + 1); @@ -1824,145 +984,78 @@ mod tokenize_into_buffer_tests { #[test] fn tokenize_into_buffer_propagates_invocation_error() { let result = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { - Err(crate::StringToTokenError::NotEnoughMemory) + Err(StringToTokenError::NotEnoughMemory) }); - assert_eq!(result, Err(crate::StringToTokenError::NotEnoughMemory)); + assert_eq!(result, Err(StringToTokenError::NotEnoughMemory)); } #[test] fn tokenize_into_buffer_propagates_second_invocation_error() { - let call_count = std::cell::Cell::new(0); + let call_count = Cell::new(0); let result = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { let count = call_count.get(); call_count.set(count + 1); if count == 0 { Ok(-20) } else { - Err(crate::StringToTokenError::NotEnoughMemory) + Err(StringToTokenError::NotEnoughMemory) } }); - assert_eq!(result, Err(crate::StringToTokenError::NotEnoughMemory)); + assert_eq!(result, Err(StringToTokenError::NotEnoughMemory)); assert_eq!(call_count.get(), 2); } #[test] - fn tokenize_into_buffer_negative_final_size_returns_conversion_error() { - let call_count = std::cell::Cell::new(0); - let result = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { - let count = call_count.get(); - call_count.set(count + 1); - if count == 0 { Ok(-20) } else { Ok(-5) } - }); - - assert_eq!( - result.unwrap_err(), - crate::StringToTokenError::CIntConversionError(usize::try_from(-5i32).unwrap_err()) - ); - } -} - -#[cfg(test)] -mod read_optional_owned_cstr_tests { - - use super::read_optional_owned_cstr; - use crate::MarkerDetectionError; - - #[test] - fn read_optional_owned_cstr_invalid_utf8_returns_error() { - let invalid_utf8_with_terminator: [u8; 3] = [0xFF, 0xFE, 0x00]; - let result = read_optional_owned_cstr( - invalid_utf8_with_terminator - .as_ptr() - .cast::(), - ); - + fn tokenize_into_buffer_refuses_a_capacity_beyond_a_c_int_before_allocating() { assert_eq!( - result.unwrap_err(), - MarkerDetectionError::MarkerUtf8Error(String::from_utf8(vec![0xFF, 0xFE]).unwrap_err()) + tokenize_into_buffer(usize::MAX, |_tokens, _n_tokens_max| Ok(0)), + Err(StringToTokenError::CIntConversionError( + i32::try_from(usize::MAX).unwrap_err() + )) ); } -} - -#[cfg(test)] -mod collect_synthetic_tool_call_renders_tests { - - use super::collect_synthetic_tool_call_renders; - use crate::MarkerDetectionError; #[test] - fn collect_synthetic_tool_call_renders_first_invalid_utf8_returns_error() { - let invalid_utf8_with_terminator: [u8; 3] = [0xFF, 0xFE, 0x00]; - let valid_with_terminator: [u8; 3] = [b'o', b'k', 0x00]; - let result = collect_synthetic_tool_call_renders( - invalid_utf8_with_terminator - .as_ptr() - .cast::(), - valid_with_terminator.as_ptr().cast::(), - ); - + fn tokenize_into_buffer_refuses_a_required_size_it_cannot_negate() { assert_eq!( - result.unwrap_err(), - MarkerDetectionError::MarkerUtf8Error(String::from_utf8(vec![0xFF, 0xFE]).unwrap_err()) + tokenize_into_buffer(8, |_tokens, _n_tokens_max| Ok(i32::MIN)), + Err(StringToTokenError::CIntConversionError( + usize::try_from(i32::MIN).unwrap_err() + )) ); } #[test] - fn collect_synthetic_tool_call_renders_second_invalid_utf8_returns_error() { - let valid_with_terminator: [u8; 3] = [b'o', b'k', 0x00]; - let invalid_utf8_with_terminator: [u8; 3] = [0xFF, 0xFE, 0x00]; - let result = collect_synthetic_tool_call_renders( - valid_with_terminator.as_ptr().cast::(), - invalid_utf8_with_terminator - .as_ptr() - .cast::(), - ); + fn tokenize_into_buffer_negative_final_size_returns_conversion_error() { + let call_count = Cell::new(0); + let result = tokenize_into_buffer(8, |_tokens, _n_tokens_max| { + let count = call_count.get(); + call_count.set(count + 1); + if count == 0 { Ok(-20) } else { Ok(-5) } + }); assert_eq!( result.unwrap_err(), - MarkerDetectionError::MarkerUtf8Error(String::from_utf8(vec![0xFF, 0xFE]).unwrap_err()) + StringToTokenError::CIntConversionError(usize::try_from(-5i32).unwrap_err()) ); } } -#[cfg(test)] -mod cxx_exception_owns_out_error_tests { - use super::cxx_exception_owns_out_error; - use crate::MarkerDetectionError; - - #[test] - fn cxx_exception_owns_out_error_classifies_each_failure_variant() { - assert!(cxx_exception_owns_out_error::<()>(&Err( - MarkerDetectionError::ReasoningMarkerDetectionFailed { - message: String::new() - } - ))); - assert!(cxx_exception_owns_out_error::<()>(&Err( - MarkerDetectionError::ToolCallHaystackComputationFailed { - message: String::new() - } - ))); - assert!(cxx_exception_owns_out_error::<()>(&Err( - MarkerDetectionError::ToolCallSyntheticRenderDiagnosisFailed { - message: String::new() - } - ))); - assert!(!cxx_exception_owns_out_error::<()>(&Ok(()))); - } -} - #[cfg(test)] mod extract_meta_string_tests { + use std::cell::Cell; + use std::slice; + use super::extract_meta_string; - use crate::MetaValError; + use crate::error::meta_val_error::MetaValError; #[test] - fn returns_error_when_null_terminator_missing() { + fn a_value_without_a_nul_terminator_is_reported_as_a_negative_return() { let result = extract_meta_string( |buf_ptr, buf_len| { - let buffer = - unsafe { std::slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; + let buffer = unsafe { slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; buffer[0] = b'a'; buffer[1] = b'b'; buffer[2] = b'c'; @@ -1975,18 +1068,17 @@ mod extract_meta_string_tests { } #[test] - fn returns_error_for_negative_return_value() { + fn a_negative_return_value_is_reported_as_is() { let result = extract_meta_string(|_buf_ptr, _buf_len| -5, 4); assert_eq!(result.unwrap_err(), MetaValError::NegativeReturn(-5)); } #[test] - fn returns_error_for_invalid_utf8_data() { + fn invalid_utf8_metadata_is_reported_as_a_utf8_error() { let result = extract_meta_string( |buf_ptr, buf_len| { - let buffer = - unsafe { std::slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; + let buffer = unsafe { slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; buffer[0] = 0xFF; buffer[1] = 0xFE; buffer[2] = 0; @@ -1995,16 +1087,17 @@ mod extract_meta_string_tests { 4, ); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("FromUtf8Error")); + let invalid_utf8 = String::from_utf8(vec![0xFF, 0xFE]).unwrap_err(); + + assert_eq!(result, Err(MetaValError::FromUtf8Error(invalid_utf8))); } #[test] - fn triggers_buffer_resize_when_returned_len_exceeds_capacity() { + fn a_value_longer_than_the_buffer_is_reread_into_a_larger_buffer() { let initial_capacity: usize = 4; let length_exceeding_initial_capacity = 10; let written_length = 2; - let call_count = std::cell::Cell::new(0); + let call_count = Cell::new(0); let result = extract_meta_string( |buf_ptr, buf_len| { let count = call_count.get(); @@ -2013,7 +1106,7 @@ mod extract_meta_string_tests { length_exceeding_initial_capacity } else { let buffer = - unsafe { std::slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; + unsafe { slice::from_raw_parts_mut(buf_ptr.cast::(), buf_len) }; buffer[0] = b'h'; buffer[1] = b'i'; buffer[2] = 0; @@ -2023,16 +1116,17 @@ mod extract_meta_string_tests { initial_capacity, ); - assert_eq!(result.unwrap(), "hi"); + assert_eq!(result, Ok("hi".to_owned())); } } #[cfg(test)] mod initialize_lora_adapter_tests { + use std::mem::ManuallyDrop; use std::ptr; use super::initialize_lora_adapter; - use crate::LlamaLoraAdapterInitError; + use crate::error::llama_lora_adapter_init_error::LlamaLoraAdapterInitError; #[test] fn lora_adapter_initialization_maps_a_null_pointer_to_unloadable() { @@ -2044,7 +1138,7 @@ mod initialize_lora_adapter_tests { #[test] fn lora_adapter_initialization_owns_a_valid_pointer() { let pointer = ptr::NonNull::dangling(); - let adapter = std::mem::ManuallyDrop::new( + let adapter = ManuallyDrop::new( initialize_lora_adapter(|| pointer.as_ptr()) .expect("a non-null adapter pointer must initialize"), ); @@ -2055,11 +1149,15 @@ mod initialize_lora_adapter_tests { #[cfg(test)] mod load_model_from_file_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::path::Path; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::load_model_from_file_status_to_result; - use crate::LlamaModelLoadError; + use crate::error::llama_model_load_error::LlamaModelLoadError; #[test] fn load_model_success_with_null_model_is_contract_error() { @@ -2074,7 +1172,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - LlamaModelLoadError::FfiContract(crate::FfiContractError { + LlamaModelLoadError::FfiContract(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "success status contained a null model", }) @@ -2114,6 +1212,27 @@ mod load_model_from_file_status_to_result_tests { assert_eq!(result.unwrap_err(), LlamaModelLoadError::NotEnoughMemory); } + #[test] + fn load_model_from_file_cxx_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"model loading threw".as_ptr()) }; + let result = unsafe { + load_model_from_file_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + Path::new("/some/path"), + ) + }; + + assert_eq!( + result.err(), + Some(LlamaModelLoadError::Reported { + message: "model loading threw".to_owned() + }) + ); + } + #[test] fn load_model_from_file_cxx_exception_without_a_message_is_a_contract_error() { let result = unsafe { @@ -2127,10 +1246,10 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_load_model_from_file", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -2148,7 +1267,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - LlamaModelLoadError::FfiStatus(crate::FfiStatusError { + LlamaModelLoadError::FfiStatus(FfiStatusError { operation: "llama_rs_load_model_from_file", code: 255, }) @@ -2156,7 +1275,7 @@ mod load_model_from_file_status_to_result_tests { } #[test] - fn load_model_from_file_status_to_result_maps_null_path_arg() { + fn load_model_from_file_status_to_result_reports_a_null_path_arg_as_a_contract_violation() { let outcome = unsafe { load_model_from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_PATH_ARG, @@ -2168,7 +1287,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null path argument", } @@ -2178,7 +1297,8 @@ mod load_model_from_file_status_to_result_tests { } #[test] - fn load_model_from_file_status_to_result_maps_null_out_model_arg() { + fn load_model_from_file_status_to_result_reports_a_null_out_model_arg_as_a_contract_violation() + { let outcome = unsafe { load_model_from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_MODEL_ARG, @@ -2190,7 +1310,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_model argument", } @@ -2200,7 +1320,8 @@ mod load_model_from_file_status_to_result_tests { } #[test] - fn load_model_from_file_status_to_result_maps_null_out_error_arg() { + fn load_model_from_file_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() + { let outcome = unsafe { load_model_from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_NULL_OUT_ERROR_ARG, @@ -2212,7 +1333,7 @@ mod load_model_from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_load_model_from_file", detail: "was given a null out_error argument", } @@ -2222,7 +1343,7 @@ mod load_model_from_file_status_to_result_tests { } #[test] - fn load_model_from_file_status_to_result_maps_llama_cpp_out_of_memory() { + fn load_model_from_file_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = unsafe { load_model_from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_LOAD_MODEL_FROM_FILE_LLAMA_CPP_OUT_OF_MEMORY, @@ -2239,1068 +1360,110 @@ mod load_model_from_file_status_to_result_tests { } #[cfg(test)] -mod chat_parser_create_status_to_result_tests { - use std::ffi::c_char; - use std::mem::discriminant; +mod tokenize_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; - use super::chat_parser_create_status_to_result; - use crate::ParseChatMessageError; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::tokenize_status_to_result; + use crate::error::string_to_token_error::StringToTokenError; #[test] - fn chat_parser_create_no_chat_template_maps_to_no_chat_template() { - let mut out_error: *mut c_char = ptr::null_mut(); + fn tokenize_ok_returns_count() { let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_CHAT_TEMPLATE, + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_OK, + 12, ptr::null_mut(), - &raw mut out_error, ) }; - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NoChatTemplate) - ); + assert_eq!(result, Ok(12)); } #[test] - fn chat_parser_create_no_vocab_maps_to_no_vocab() { - let mut out_error: *mut c_char = ptr::null_mut(); + fn tokenize_allocation_failed_is_not_enough_memory() { let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_MODEL_HAS_NO_VOCAB, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NoVocab) - ); - } - - #[test] - fn chat_parser_create_allocation_failed_is_not_enough_memory() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::NotEnoughMemory) - ); - } - - #[test] - fn chat_parser_create_cxx_exception_is_parser_creation_failed_and_nulls_error() { - let mut out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the parser could not be built".as_ptr()) - }; - let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::ParserCreationFailed { - message: "the parser could not be built".to_owned() - }) - ); - - assert!( - out_error.is_null(), - "the reclaimed pointer must be nulled so the caller does not free it twice" - ); - } - - #[test] - fn chat_parser_create_success_with_null_parser_is_contract_error() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_OK, - ptr::null_mut(), - &raw mut out_error, - ) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "success status contained a null parser handle", - } - )) - ); - } - - #[test] - fn chat_parser_create_unknown_status_is_preserved() { - let mut out_error: *mut c_char = ptr::null_mut(); - let result = unsafe { - chat_parser_create_status_to_result(255, ptr::null_mut(), &raw mut out_error) - }; - - assert_eq!( - discriminant(&result.unwrap_err()), - discriminant(&ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_parser_create", - code: 255, - })) - ); - } - - #[test] - fn chat_parser_create_status_to_result_maps_null_model_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_MODEL_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null model argument", - } - )) - ); - } - - #[test] - fn chat_parser_create_status_to_result_maps_null_out_parser_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_PARSER_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null out_parser argument", - } - )) - ); - } - - #[test] - fn chat_parser_create_status_to_result_maps_null_out_error_arg() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_NULL_OUT_ERROR_ARG, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_chat_parser_create", - detail: "was given a null out_error argument", - } - )) - ); - } - - #[test] - fn chat_parser_create_status_to_result_maps_llama_cpp_out_of_memory() { - let mut out_error_slot: *mut c_char = ptr::null_mut(); - let outcome = unsafe { - chat_parser_create_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - &raw mut out_error_slot, - ) - }; - assert_eq!( - outcome.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod detect_reasoning_markers_status_to_result_tests { - use std::ptr; - - use super::detect_reasoning_markers_status_to_result; - use crate::MarkerDetectionError; - - #[test] - fn detect_reasoning_markers_ok_with_null_handle_returns_none() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_OK, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok(None)); - } - - #[test] - fn detect_reasoning_markers_allocation_failed_is_not_enough_memory() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(MarkerDetectionError::NotEnoughMemory)); - } - - #[test] - fn detect_reasoning_markers_cxx_exception_without_a_message_is_a_contract_error() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_detect_reasoning_markers", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn detect_reasoning_markers_unknown_status_is_preserved() { - let result = - unsafe { detect_reasoning_markers_status_to_result(255, ptr::null(), ptr::null_mut()) }; - - assert_eq!( - result, - Err(MarkerDetectionError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_detect_reasoning_markers", - code: 255, - })) - ); - } - - #[test] - fn detect_reasoning_markers_null_model_status_names_model_argument() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_MODEL_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "model", - }) - ); - } - - #[test] - fn detect_reasoning_markers_null_output_status_names_markers_argument() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_MARKERS_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "out_markers", - }) - ); - } - - #[test] - fn detect_reasoning_markers_null_error_status_names_error_argument() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::WrapperRejectedArgument { - operation: "llama_rs_detect_reasoning_markers", - argument: "out_error", - }) - ); - } - - #[test] - fn detect_reasoning_markers_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(MarkerDetectionError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod validate_reasoning_markers_tests { - use llama_cpp_bindings_types::ReasoningMarkers; - - use super::validate_reasoning_markers; - use crate::MarkerDetectionError; - - #[test] - fn reasoning_markers_reject_a_missing_opening_marker() { - let result = validate_reasoning_markers(None, vec![Some("".to_owned())]); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_open", - detail: "non-null markers returned a null opening marker", - })) - ); - } - - #[test] - fn reasoning_markers_reject_an_empty_opening_marker() { - let result = - validate_reasoning_markers(Some(String::new()), vec![Some("".to_owned())]); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_detect_reasoning_markers", - detail: "detected markers must contain an opening marker and a closing marker", - })) - ); - } - - #[test] - fn reasoning_markers_reject_an_empty_closing_marker_collection() { - let result = validate_reasoning_markers(Some("".to_owned()), Vec::new()); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_detect_reasoning_markers", - detail: "detected markers must contain an opening marker and a closing marker", - })) - ); - } - - #[test] - fn reasoning_markers_reject_a_missing_closing_marker() { - let result = validate_reasoning_markers(Some("".to_owned()), vec![None]); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_close_at", - detail: "a valid closing-marker index returned null", - })) - ); - } - - #[test] - fn reasoning_markers_reject_an_empty_closing_marker() { - let result = - validate_reasoning_markers(Some("".to_owned()), vec![Some(String::new())]); - - assert_eq!( - result, - Err(MarkerDetectionError::FfiContract(crate::FfiContractError { - operation: "llama_rs_reasoning_markers_close_at", - detail: "a detected closing marker was empty", - })) - ); - } - - #[test] - fn reasoning_markers_preserve_every_valid_closing_marker() { - let result = validate_reasoning_markers( - Some("".to_owned()), - vec![Some("".to_owned()), Some("".to_owned())], - ); - - assert_eq!( - result, - Ok(ReasoningMarkers { - open: "".to_owned(), - closes: vec!["".to_owned(), "".to_owned()], - }) - ); - } -} - -#[cfg(test)] -mod compute_tool_call_haystack_status_to_result_tests { - use std::ptr; - - use super::compute_tool_call_haystack_status_to_result; - use crate::MarkerDetectionError; - - #[test] - fn compute_tool_call_haystack_ok_with_null_pointer_is_none() { - let result = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_OK, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok(None)); - } - - #[test] - fn compute_tool_call_haystack_allocation_failed_is_not_enough_memory() { - let result = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_ERROR_STRING_ALLOCATION_FAILED, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(MarkerDetectionError::NotEnoughMemory)); - } - - #[test] - fn compute_tool_call_haystack_cxx_exception_without_a_message_is_a_contract_error() { - let result = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn compute_tool_call_haystack_unknown_status_is_preserved() { - let result = unsafe { - compute_tool_call_haystack_status_to_result(255, ptr::null(), ptr::null_mut()) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_compute_tool_call_haystack", - code: 255, - })) - ); - } - - #[test] - fn compute_tool_call_haystack_status_to_result_maps_null_model_arg() { - let outcome = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_MODEL_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null model argument", - } - .into() - ) - ); - } - - #[test] - fn compute_tool_call_haystack_status_to_result_maps_null_out_haystack_arg() { - let outcome = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_HAYSTACK_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null out_haystack argument", - } - .into() - ) - ); - } - - #[test] - fn compute_tool_call_haystack_status_to_result_maps_null_out_error_arg() { - let outcome = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_NULL_OUT_ERROR_ARG, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_compute_tool_call_haystack", - detail: "was given a null out_error argument", - } - .into() - ) - ); - } - - #[test] - fn compute_tool_call_haystack_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - compute_tool_call_haystack_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_COMPUTE_TOOL_CALL_HAYSTACK_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(MarkerDetectionError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod diagnose_tool_call_synthetic_renders_status_to_result_tests { - use std::ptr; - - use super::diagnose_tool_call_synthetic_renders_status_to_result; - use crate::MarkerDetectionError; - - #[test] - fn diagnose_tool_call_synthetic_renders_ok_with_null_pointers_is_a_contract_error() { - let result = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_OK, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "success status contained a null render without tools", - } - .into()) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_allocation_failed_is_not_enough_memory() { - let result = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_ERROR_STRING_ALLOCATION_FAILED, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(MarkerDetectionError::NotEnoughMemory)); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_cxx_exception_without_a_message_is_a_contract_error() { - let result = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_unknown_status_is_preserved() { - let result = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result( - 255, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - code: 255, - })) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_null_model_arg() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_MODEL_ARG, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null model argument", - } - .into() - ) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_null_out_no_tools_arg() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_NO_TOOLS_ARG, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_no_tools argument", - } - .into() - ) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_null_out_with_tools_arg() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_WITH_TOOLS_ARG, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_with_tools argument", - } - .into() - ) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_null_out_error_arg() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_NULL_OUT_ERROR_ARG, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_diagnose_tool_call_synthetic_renders", - detail: "was given a null out_error argument", - } - .into() - ) - ); - } - - #[test] - fn diagnose_tool_call_synthetic_renders_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { - diagnose_tool_call_synthetic_renders_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_DIAGNOSE_TOOL_CALL_SYNTHETIC_RENDERS_LLAMA_CPP_OUT_OF_MEMORY, ptr::null(), ptr::null(), ptr::null_mut()) - }; - assert_eq!( - outcome.err(), - Some(MarkerDetectionError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod tokenize_status_to_result_tests { - use std::ptr; - - use super::tokenize_status_to_result; - use crate::StringToTokenError; - - #[test] - fn tokenize_ok_returns_count() { - let result = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_OK, - 12, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok(12)); - } - - #[test] - fn tokenize_allocation_failed_is_not_enough_memory() { - let result = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_ERROR_STRING_ALLOCATION_FAILED, - 0, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(StringToTokenError::NotEnoughMemory)); - } - - #[test] - fn tokenize_cxx_exception_without_a_message_is_a_contract_error() { - let result = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn tokenize_unknown_status_is_preserved() { - let result = unsafe { tokenize_status_to_result(255, 0, ptr::null_mut()) }; - - assert_eq!( - result, - Err(StringToTokenError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_tokenize", - code: 255, - })) - ); - } - - #[test] - fn tokenize_status_to_result_maps_null_vocab_arg() { - let outcome = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null vocab argument", - } - .into() - ) - ); - } - - #[test] - fn tokenize_status_to_result_maps_null_text_arg() { - let outcome = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null text argument", - } - .into() - ) - ); - } - - #[test] - fn tokenize_status_to_result_maps_null_out_returned_count_arg() { - let outcome = unsafe { tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_RETURNED_COUNT_ARG, + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_ERROR_STRING_ALLOCATION_FAILED, 0, ptr::null_mut(), ) }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null out_returned_count argument", - } - .into() - ) - ); - } - #[test] - fn tokenize_status_to_result_maps_null_out_error_arg() { - let outcome = unsafe { - tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_ERROR_ARG, - 0, - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_tokenize", - detail: "was given a null out_error argument", - } - .into() - ) - ); + assert_eq!(result, Err(StringToTokenError::NotEnoughMemory)); } #[test] - fn tokenize_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = unsafe { + fn tokenize_cxx_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"tokenizer threw".as_ptr()) }; + let result = unsafe { tokenize_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, 0, - ptr::null_mut(), - ) - }; - assert_eq!(outcome.err(), Some(StringToTokenError::LlamaCppOutOfMemory)); - } -} - -#[cfg(test)] -mod apply_chat_template_status_to_result_tests { - use std::ptr; - - use super::apply_chat_template_status_to_result; - use crate::error::apply_chat_template_error::ApplyChatTemplateError; - - #[test] - fn apply_chat_template_ok_returns_rendered_prompt() { - let rendered = std::ffi::CString::new("rendered prompt").unwrap(); - let out_string = unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(rendered.as_ptr()) }; - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_OK, - out_string, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok("rendered prompt".to_owned())); - } - - #[test] - fn apply_chat_template_ok_with_null_rendered_prompt_is_contract_error() { - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_OK, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!( - result, - Err(ApplyChatTemplateError::FfiContract( - crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "success status contained a null rendered string", - } - )) - ); - } - - #[test] - fn apply_chat_template_no_vocab_maps_to_no_vocab() { - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_MODEL_HAS_NO_VOCAB, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(ApplyChatTemplateError::NoVocab)); - } - - #[test] - fn apply_chat_template_application_failed_maps_to_template_application_failed() { - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_TEMPLATE_APPLICATION_FAILED, - ptr::null_mut(), - ptr::null_mut(), + out_error, ) }; assert_eq!( result, - Err(ApplyChatTemplateError::TemplateApplicationFailed) + Err(StringToTokenError::Reported { + message: "tokenizer threw".to_owned() + }) ); } #[test] - fn apply_chat_template_allocation_failed_maps_to_not_enough_memory() { - let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(ApplyChatTemplateError::NotEnoughMemory)); - } - - #[test] - fn apply_chat_template_cxx_exception_is_reported() { - let message = std::ffi::CString::new("renderer exploded").unwrap(); - let out_error = unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(message.as_ptr()) }; + fn tokenize_cxx_exception_without_a_message_is_a_contract_error() { let result = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_THREW_CXX_EXCEPTION, + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, ptr::null_mut(), - out_error, ) }; assert_eq!( result, - Err(ApplyChatTemplateError::Reported { - message: "renderer exploded".to_owned() + Err(CppStringError::Missing(FfiContractError { + operation: "llama_rs_tokenize", + detail: "reported a thrown C++ exception without an error message", }) + .into()) ); } #[test] - fn apply_chat_template_unknown_status_is_preserved() { - let result = - unsafe { apply_chat_template_status_to_result(255, ptr::null_mut(), ptr::null_mut()) }; + fn tokenize_unknown_status_is_preserved() { + let result = unsafe { tokenize_status_to_result(255, 0, ptr::null_mut()) }; assert_eq!( result, - Err(ApplyChatTemplateError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_apply_chat_template", + Err(StringToTokenError::FfiStatus(FfiStatusError { + operation: "llama_rs_tokenize", code: 255, })) ); } #[test] - fn apply_chat_template_status_to_result_maps_null_model_arg() { - let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MODEL_ARG, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null model argument", - } - .into() - ) - ); - } - - #[test] - fn apply_chat_template_status_to_result_maps_null_template_arg() { + fn tokenize_status_to_result_reports_a_null_vocab_arg_as_a_contract_violation() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_TEMPLATE_ARG, - ptr::null_mut(), + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_VOCAB_ARG, + 0, ptr::null_mut(), ) }; assert_eq!( outcome.err(), Some( - crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null template argument", + FfiContractError { + operation: "llama_rs_tokenize", + detail: "was given a null vocab argument", } .into() ) @@ -3308,20 +1471,20 @@ mod apply_chat_template_status_to_result_tests { } #[test] - fn apply_chat_template_status_to_result_maps_null_messages_arg() { + fn tokenize_status_to_result_reports_a_null_text_arg_as_a_contract_violation() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_MESSAGES_ARG, - ptr::null_mut(), + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_TEXT_ARG, + 0, ptr::null_mut(), ) }; assert_eq!( outcome.err(), Some( - crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null messages argument", + FfiContractError { + operation: "llama_rs_tokenize", + detail: "was given a null text argument", } .into() ) @@ -3329,20 +1492,20 @@ mod apply_chat_template_status_to_result_tests { } #[test] - fn apply_chat_template_status_to_result_maps_null_out_string_arg() { + fn tokenize_status_to_result_reports_a_null_out_returned_count_arg_as_a_contract_violation() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_STRING_ARG, - ptr::null_mut(), + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_RETURNED_COUNT_ARG, + 0, ptr::null_mut(), ) }; assert_eq!( outcome.err(), Some( - crate::FfiContractError { - operation: "llama_rs_apply_chat_template", - detail: "was given a null out_string argument", + FfiContractError { + operation: "llama_rs_tokenize", + detail: "was given a null out_returned_count argument", } .into() ) @@ -3350,19 +1513,19 @@ mod apply_chat_template_status_to_result_tests { } #[test] - fn apply_chat_template_status_to_result_maps_null_out_error_arg() { + fn tokenize_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_NULL_OUT_ERROR_ARG, - ptr::null_mut(), + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_NULL_OUT_ERROR_ARG, + 0, ptr::null_mut(), ) }; assert_eq!( outcome.err(), Some( - crate::FfiContractError { - operation: "llama_rs_apply_chat_template", + FfiContractError { + operation: "llama_rs_tokenize", detail: "was given a null out_error argument", } .into() @@ -3371,178 +1534,14 @@ mod apply_chat_template_status_to_result_tests { } #[test] - fn apply_chat_template_status_to_result_maps_llama_cpp_out_of_memory() { + fn tokenize_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = unsafe { - apply_chat_template_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_APPLY_CHAT_TEMPLATE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ptr::null_mut(), - ) - }; - assert_eq!( - outcome.err(), - Some(ApplyChatTemplateError::LlamaCppOutOfMemory) - ); - } -} - -#[cfg(test)] -mod chat_parser_free_status_to_result_tests { - use std::ptr; - - use super::chat_parser_free_status_to_result; - use crate::ParseChatMessageError; - - #[test] - fn chat_parser_free_ok_is_success() { - let result = unsafe { - chat_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_OK, - ptr::null_mut(), - ) - }; - - assert!( - result.is_ok(), - "a clean destructor must not report a failure" - ); - } - - #[test] - fn chat_parser_free_allocation_failed_is_not_enough_memory() { - let result = unsafe { - chat_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ) - }; - - assert_eq!(result.err(), Some(ParseChatMessageError::NotEnoughMemory)); - } - - #[test] - fn chat_parser_free_llama_cpp_out_of_memory_is_preserved() { - let result = unsafe { - chat_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::LlamaCppOutOfMemory) - ); - } - - #[test] - fn chat_parser_free_destructor_threw_surfaces_the_message() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the destructor threw".as_ptr()) - }; - let result = unsafe { - chat_parser_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_CHAT_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - out_error, - ) - }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::DestructorFailed { - message: "the destructor threw".to_owned() - }) - ); - } - - #[test] - fn chat_parser_free_unknown_status_is_preserved() { - let result = unsafe { chat_parser_free_status_to_result(255, ptr::null_mut()) }; - - assert_eq!( - result.err(), - Some(ParseChatMessageError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_chat_parser_free", - code: 255, - })) - ); - } -} - -#[cfg(test)] -mod reasoning_markers_free_status_to_result_tests { - use std::ptr; - - use super::reasoning_markers_free_status_to_result; - use crate::MarkerDetectionError; - - #[test] - fn reasoning_markers_free_ok_is_success() { - let result = unsafe { - reasoning_markers_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_OK, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Ok(())); - } - - #[test] - fn reasoning_markers_free_allocation_failed_is_not_enough_memory() { - let result = unsafe { - reasoning_markers_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ) - }; - - assert_eq!(result, Err(MarkerDetectionError::NotEnoughMemory)); - } - - #[test] - fn reasoning_markers_free_llama_cpp_out_of_memory_is_preserved() { - let result = unsafe { - reasoning_markers_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_LLAMA_CPP_OUT_OF_MEMORY, + tokenize_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_TOKENIZE_LLAMA_CPP_OUT_OF_MEMORY, + 0, ptr::null_mut(), ) }; - - assert_eq!(result, Err(MarkerDetectionError::LlamaCppOutOfMemory)); - } - - #[test] - fn reasoning_markers_free_destructor_threw_surfaces_the_message() { - let out_error = unsafe { - llama_cpp_bindings_sys::llama_rs_string_dup(c"the destructor threw".as_ptr()) - }; - let result = unsafe { - reasoning_markers_free_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_REASONING_MARKERS_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION, - out_error, - ) - }; - - assert_eq!( - result, - Err(MarkerDetectionError::ReasoningMarkersFreeFailed { - message: "the destructor threw".to_owned(), - }) - ); - } - - #[test] - fn reasoning_markers_free_unknown_status_is_preserved() { - let result = unsafe { reasoning_markers_free_status_to_result(255, ptr::null_mut()) }; - - assert_eq!( - result, - Err(crate::FfiStatusError { - operation: "llama_rs_reasoning_markers_free", - code: 255, - } - .into()) - ); + assert_eq!(outcome.err(), Some(StringToTokenError::LlamaCppOutOfMemory)); } } diff --git a/llama-cpp-bindings/src/model/llama_chat_message.rs b/llama-cpp-bindings/src/model/llama_chat_message.rs index 51e1f0867..f692f4eca 100644 --- a/llama-cpp-bindings/src/model/llama_chat_message.rs +++ b/llama-cpp-bindings/src/model/llama_chat_message.rs @@ -1,6 +1,6 @@ use std::ffi::CString; -use crate::NewLlamaChatMessageError; +use crate::error::new_llama_chat_message_error::NewLlamaChatMessageError; #[derive(Debug, Eq, PartialEq, Clone)] pub struct LlamaChatMessage { @@ -21,33 +21,39 @@ impl LlamaChatMessage { #[cfg(test)] mod tests { - use super::LlamaChatMessage; + use std::ffi::CString; - #[test] - fn valid_construction() { - let message = LlamaChatMessage::new("user".to_string(), "hello".to_string()); - - assert!(message.is_ok()); - } + use crate::error::new_llama_chat_message_error::NewLlamaChatMessageError; + use crate::model::llama_chat_message::LlamaChatMessage; #[test] - fn null_byte_in_role_returns_error() { - let message = LlamaChatMessage::new("us\0er".to_string(), "hello".to_string()); - - assert!(message.is_err()); + fn a_role_and_content_without_nul_bytes_form_a_message() { + assert_eq!( + LlamaChatMessage::new("user".to_string(), "hello".to_string()), + Ok(LlamaChatMessage { + role: c"user".to_owned(), + content: c"hello".to_owned(), + }) + ); } #[test] - fn null_byte_in_content_returns_error() { - let message = LlamaChatMessage::new("user".to_string(), "hel\0lo".to_string()); - - assert!(message.is_err()); + fn a_nul_byte_in_the_role_is_reported() { + assert_eq!( + LlamaChatMessage::new("us\0er".to_string(), "hello".to_string()), + Err(NewLlamaChatMessageError::NulError( + CString::new("us\0er").unwrap_err() + )) + ); } #[test] - fn empty_strings_are_valid() { - let message = LlamaChatMessage::new(String::new(), String::new()); - - assert!(message.is_ok()); + fn a_nul_byte_in_the_content_is_reported() { + assert_eq!( + LlamaChatMessage::new("user".to_string(), "hel\0lo".to_string()), + Err(NewLlamaChatMessageError::NulError( + CString::new("hel\0lo").unwrap_err() + )) + ); } } diff --git a/llama-cpp-bindings/src/model/llama_chat_template.rs b/llama-cpp-bindings/src/model/llama_chat_template.rs index 3e8f86d02..c06125cfb 100644 --- a/llama-cpp-bindings/src/model/llama_chat_template.rs +++ b/llama-cpp-bindings/src/model/llama_chat_template.rs @@ -1,4 +1,8 @@ +use std::ffi::NulError; use std::ffi::{CStr, CString}; +use std::fmt; +use std::fmt::Debug; +use std::fmt::Formatter; use std::str::Utf8Error; #[derive(Eq, PartialEq, Clone, PartialOrd, Ord, Hash)] @@ -7,7 +11,7 @@ pub struct LlamaChatTemplate(pub CString); impl LlamaChatTemplate { /// # Errors /// Returns an error if the template string contains null bytes. - pub fn new(template: &str) -> Result { + pub fn new(template: &str) -> Result { Ok(Self(CString::new(template)?)) } @@ -29,18 +33,20 @@ impl LlamaChatTemplate { } } -impl std::fmt::Debug for LlamaChatTemplate { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl Debug for LlamaChatTemplate { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { self.0.fmt(f) } } #[cfg(test)] mod tests { - use super::LlamaChatTemplate; + use std::ffi::CString; + + use crate::model::llama_chat_template::LlamaChatTemplate; #[test] - fn valid_template_creation() { + fn a_template_keeps_its_name() { let template = LlamaChatTemplate::new("chatml").unwrap(); let template_str = template.to_str().unwrap(); @@ -48,22 +54,23 @@ mod tests { } #[test] - fn null_byte_returns_error() { - let template = LlamaChatTemplate::new("null\0byte"); - - assert!(template.is_err()); + fn a_nul_byte_in_the_template_is_reported() { + assert_eq!( + LlamaChatTemplate::new("null\0byte"), + Err(CString::new("null\0byte").unwrap_err()) + ); } #[test] - fn debug_formatting() { + fn debug_formats_a_template_as_its_quoted_name() { let template = LlamaChatTemplate::new("chatml").unwrap(); let debug_output = format!("{template:?}"); - assert!(debug_output.contains("chatml")); + assert_eq!(debug_output, "\"chatml\""); } #[test] - fn to_string_returns_owned_string() { + fn to_string_copies_the_template_name() { let template = LlamaChatTemplate::new("llama3").unwrap(); let owned = template.to_string().unwrap(); @@ -71,7 +78,7 @@ mod tests { } #[test] - fn as_c_str_returns_valid_cstr() { + fn as_c_str_views_the_template_name() { let template = LlamaChatTemplate::new("test").unwrap(); let cstr = template.as_c_str(); diff --git a/llama-cpp-bindings/src/model/llama_lazy_mode.rs b/llama-cpp-bindings/src/model/llama_lazy_mode.rs index c3745c784..f234acfab 100644 --- a/llama-cpp-bindings/src/model/llama_lazy_mode.rs +++ b/llama-cpp-bindings/src/model/llama_lazy_mode.rs @@ -35,7 +35,7 @@ impl From for llama_cpp_bindings_sys::llama_lazy_mode { #[cfg(test)] mod tests { - use super::LlamaLazyMode; + use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; const LAZY_MODES: &[(LlamaLazyMode, llama_cpp_bindings_sys::llama_lazy_mode)] = &[ diff --git a/llama-cpp-bindings/src/model/llama_load_mode.rs b/llama-cpp-bindings/src/model/llama_load_mode.rs index 546175103..41b01b44b 100644 --- a/llama-cpp-bindings/src/model/llama_load_mode.rs +++ b/llama-cpp-bindings/src/model/llama_load_mode.rs @@ -41,7 +41,8 @@ impl From for llama_cpp_bindings_sys::llama_load_mode { #[cfg(test)] mod tests { - use super::LlamaLoadMode; + use crate::model::llama_load_mode::LlamaLoadMode; + use crate::model::llama_load_mode_parse_error::LlamaLoadModeParseError; const LOAD_MODES: &[(LlamaLoadMode, llama_cpp_bindings_sys::llama_load_mode)] = &[ ( @@ -91,7 +92,7 @@ mod tests { fn unknown_ffi_load_mode_preserves_its_value() { assert_eq!( LlamaLoadMode::try_from(i32::MAX), - Err(super::LlamaLoadModeParseError { value: i32::MAX }) + Err(LlamaLoadModeParseError { value: i32::MAX }) ); } } diff --git a/llama-cpp-bindings/src/model/llama_lora_adapter.rs b/llama-cpp-bindings/src/model/llama_lora_adapter.rs index d0a4484c4..2183446bc 100644 --- a/llama-cpp-bindings/src/model/llama_lora_adapter.rs +++ b/llama-cpp-bindings/src/model/llama_lora_adapter.rs @@ -14,7 +14,14 @@ pub struct LlamaLoraAdapter<'model> { } impl LlamaLoraAdapter<'_> { - pub(crate) const fn new( + /// Takes ownership of an adapter llama.cpp initialized, and frees it on drop. + /// + /// # Safety + /// + /// `lora_adapter` must come from `llama_adapter_lora_init` for the model this adapter + /// borrows, and no other [`LlamaLoraAdapter`] may own it. + #[must_use] + pub const unsafe fn new( lora_adapter: NonNull, ) -> Self { Self { @@ -23,7 +30,8 @@ impl LlamaLoraAdapter<'_> { } } - pub(crate) const fn as_ptr(&self) -> *mut llama_cpp_bindings_sys::llama_adapter_lora { + #[must_use] + pub const fn as_ptr(&self) -> *mut llama_cpp_bindings_sys::llama_adapter_lora { self.lora_adapter } } @@ -37,23 +45,14 @@ impl Drop for LlamaLoraAdapter<'_> { #[cfg(test)] mod ownership_tests { use std::marker::PhantomData; - use std::mem::ManuallyDrop; - use std::ptr::NonNull; + use std::ptr; - use super::LlamaLoraAdapter; + use crate::model::llama_lora_adapter::LlamaLoraAdapter; #[test] - fn adapter_preserves_the_owned_native_pointer() { - let pointer = NonNull::dangling(); - let adapter = ManuallyDrop::new(LlamaLoraAdapter::new(pointer)); - - assert_eq!(adapter.as_ptr(), pointer.as_ptr()); - } - - #[test] - fn dropping_an_adapter_releases_its_native_pointer() { + fn dropping_an_adapter_holding_a_null_pointer_is_tolerated() { drop(LlamaLoraAdapter { - lora_adapter: std::ptr::null_mut(), + lora_adapter: ptr::null_mut(), model: PhantomData, }); } diff --git a/llama-cpp-bindings/src/model/lora_adapter_scale.rs b/llama-cpp-bindings/src/model/lora_adapter_scale.rs index beef995b7..0add5e957 100644 --- a/llama-cpp-bindings/src/model/lora_adapter_scale.rs +++ b/llama-cpp-bindings/src/model/lora_adapter_scale.rs @@ -1,4 +1,4 @@ -use crate::model::LlamaLoraAdapter; +use crate::model::llama_lora_adapter::LlamaLoraAdapter; #[derive(Debug)] pub struct LoraAdapterScale<'adapter, 'model> { diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index 8265a7e96..78c2ca188 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -1,22 +1,32 @@ -use std::ffi::{CStr, c_char}; +use std::ffi::{CStr, CString, c_char}; +use std::fmt; use std::fmt::{Debug, Formatter}; use std::pin::Pin; +use std::ptr; use std::ptr::null; -use crate::LlamaCppError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + +use crate::backend_device_index::backend_device_index; use crate::context::params::LlamaContextParams; -use crate::error::{FitError, ModelParamsError}; +use crate::error::fit_error::FitError; +use crate::error::llama_cpp_error::LlamaCppError; +use crate::error::model_params_error::ModelParamsError; +use crate::max_devices::max_devices; use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; use crate::model::llama_load_mode::LlamaLoadMode; use crate::model::llama_load_mode_parse_error::LlamaLoadModeParseError; use crate::model::llama_split_mode_parse_error::LlamaSplitModeParseError; +use crate::model::params::empty_kv_override::EMPTY_KV_OVERRIDE; use crate::model::params::fit_extra_model::FitExtraModel; use crate::model::params::fit_result::FitResult; use crate::model::params::kv_overrides::KvOverrides; use crate::model::split_mode::LlamaSplitMode; -use llama_cpp_ffi_status::read_and_free_cpp_string; +pub mod empty_kv_override; pub mod fit_extra_model; pub mod fit_result; pub mod kv_override_entry; @@ -25,8 +35,6 @@ pub mod kv_overrides; pub mod param_override_value; pub mod unknown_kv_override_tag; -pub const LLAMA_CPP_MAX_DEVICES: usize = 16; - fn fit_params_status_to_result( status: llama_cpp_bindings_sys::llama_rs_fit_params_status, out_unrecognized_status_code: i32, @@ -61,63 +69,55 @@ fn fit_params_status_to_result( }?; Err(FitError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null path_model argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG => Err(FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given a null path_model argument", } - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null mparams argument", - } - .into()) + .into()), + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG => Err(FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given a null mparams argument", } - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null cparams argument", - } - .into()) + .into()), + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG => Err(FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given a null cparams argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_UNRECOGNIZED_STATUS_CODE_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_fit_params", detail: "was given a null out_unrecognized_status_code argument", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_fit_params", - detail: "was given a null out_error argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG => Err(FfiContractError { + operation: "llama_rs_fit_params", + detail: "was given a null out_error argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_fit_params", detail: "was given an extra model with a null path_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_fit_params", detail: "was given an extra model with a null mparams argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_fit_params", detail: "was given an extra model with a null cparams argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_fit_params", code: i64::from(other), } @@ -129,12 +129,13 @@ pub struct LlamaModelParams { pub params: llama_cpp_bindings_sys::llama_model_params, kv_overrides: Vec, buft_overrides: Vec, - devices: Pin>, + buft_override_patterns: Vec, + devices: Pin>, tensor_split: Vec, } impl Debug for LlamaModelParams { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("LlamaModelParams") .field("n_gpu_layers", &self.params.n_gpu_layers) .field("main_gpu", &self.params.main_gpu) @@ -143,7 +144,7 @@ impl Debug for LlamaModelParams { .field("load_mode", &self.load_mode()) .field("load_mtp", &self.params.load_mtp) .field("split_mode", &self.split_mode()) - .field("devices", &self.devices) + .field("devices", &self.devices()) .field("kv_overrides", &"vec of kv_overrides") .finish_non_exhaustive() } @@ -173,13 +174,17 @@ impl LlamaModelParams { return Err(ModelParamsError::SlotNotEmpty); } - for (i, &byte) in key.to_bytes_with_nul().iter().enumerate() { - kv_override.key[i] = c_char::try_from(byte).map_err(|convert_error| { - ModelParamsError::InvalidCharacterInKey { - byte, - reason: convert_error.to_string(), - } - })?; + let key_with_nul = key.to_bytes_with_nul(); + + if key_with_nul.len() > kv_override.key.len() { + return Err(ModelParamsError::KeyTooLong { + length_with_nul: key_with_nul.len(), + capacity: kv_override.key.len(), + }); + } + + for (slot, &byte) in kv_override.key.iter_mut().zip(key_with_nul) { + *slot = c_char::from_ne_bytes([byte]); } kv_override.tag = value.tag(); @@ -193,14 +198,7 @@ impl LlamaModelParams { fn push_kv_override_terminator(mut self: Pin<&mut Self>) { self.params.kv_overrides = null(); - self.kv_overrides - .push(llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], - tag: 0, - __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { - val_i64: 0, - }, - }); + self.kv_overrides.push(EMPTY_KV_OVERRIDE); self.params.kv_overrides = self.kv_overrides.as_ptr(); } @@ -230,18 +228,12 @@ impl LlamaModelParams { return Err(ModelParamsError::SlotNotEmpty); } - for &byte in key.to_bytes_with_nul() { - c_char::try_from(byte).map_err(|convert_error| { - ModelParamsError::InvalidCharacterInKey { - byte, - reason: convert_error.to_string(), - } - })?; - } + let pattern = key.to_owned(); - buft_override.pattern = key.as_ptr(); + buft_override.pattern = pattern.as_ptr(); buft_override.buft = unsafe { llama_cpp_bindings_sys::ggml_backend_cpu_buffer_type() }; + self.buft_override_patterns.push(pattern); self.push_buft_override_terminator(); Ok(()) @@ -253,7 +245,7 @@ impl LlamaModelParams { self.buft_overrides .push(llama_cpp_bindings_sys::llama_model_tensor_buft_override { pattern: null(), - buft: std::ptr::null_mut(), + buft: ptr::null_mut(), }); self.params.tensor_buft_overrides = self.buft_overrides.as_ptr(); @@ -301,27 +293,11 @@ impl LlamaModelParams { #[must_use] pub fn devices(&self) -> Vec { - let mut backend_devices = Vec::new(); - for device_index in 0..unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() } { - let dev = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) }; - backend_devices.push(dev); - } - let mut devices = Vec::new(); - for &dev in self.devices.iter() { - if dev.is_null() { - break; - } - let matched_index = backend_devices - .iter() - .enumerate() - .find(|&(_i, &d)| d == dev) - .map(|(index, _)| index); - - if let Some(index) = matched_index { - devices.push(index); - } - } - devices + self.devices + .iter() + .take_while(|device| !device.is_null()) + .filter_map(|device| backend_device_index(*device)) + .collect() } #[must_use] @@ -381,9 +357,9 @@ impl LlamaModelParams { /// Returns `LlamaCppError::BackendDeviceNotFound` if any device index is invalid. pub fn with_devices(mut self, devices: &[usize]) -> Result { for dev in self.devices.iter_mut() { - *dev = std::ptr::null_mut(); + *dev = ptr::null_mut(); } - let max_devices = crate::max_devices().min(LLAMA_CPP_MAX_DEVICES); + let max_devices = max_devices(); if devices.len() > max_devices { return Err(LlamaCppError::MaxDevicesExceeded(max_devices)); } @@ -416,6 +392,13 @@ impl LlamaModelParams { let max_devices = unsafe { llama_cpp_bindings_sys::llama_max_devices() }; let max_buft = unsafe { llama_cpp_bindings_sys::llama_max_tensor_buft_overrides() }; + if margins.len() != max_devices { + return Err(FitError::MarginsDeviceCountMismatch { + expected: max_devices, + actual: margins.len(), + }); + } + self.tensor_split.clear(); self.tensor_split.resize(max_devices, 0.0); @@ -424,7 +407,7 @@ impl LlamaModelParams { max_buft + 1, llama_cpp_bindings_sys::llama_model_tensor_buft_override { pattern: null(), - buft: std::ptr::null_mut(), + buft: ptr::null_mut(), }, ); @@ -432,7 +415,7 @@ impl LlamaModelParams { self.params.tensor_buft_overrides = null(); let mut out_unrecognized_status_code: i32 = 0; - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let mut extra_ffi = extra.as_mut().map(|extra_model| extra_model.as_ffi()); let extra_ptr = extra_ffi @@ -471,18 +454,13 @@ impl Default for LlamaModelParams { let default_params = unsafe { llama_cpp_bindings_sys::llama_model_default_params() }; Self { params: default_params, - kv_overrides: vec![llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], - tag: 0, - __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { - val_i64: 0, - }, - }], + kv_overrides: vec![EMPTY_KV_OVERRIDE], buft_overrides: vec![llama_cpp_bindings_sys::llama_model_tensor_buft_override { pattern: null(), - buft: std::ptr::null_mut(), + buft: ptr::null_mut(), }], - devices: Box::pin([std::ptr::null_mut(); 16]), + buft_override_patterns: Vec::new(), + devices: Pin::new(vec![ptr::null_mut(); max_devices() + 1].into_boxed_slice()), tensor_split: Vec::new(), } } @@ -490,44 +468,27 @@ impl Default for LlamaModelParams { #[cfg(test)] mod tests { + use crate::model::params::empty_kv_override::KV_OVERRIDE_TEXT_CAPACITY; + use std::ffi::CStr; + use std::ffi::CString; + use std::pin::pin; use std::ptr; - use super::LLAMA_CPP_MAX_DEVICES; use super::LlamaModelParams; - use super::fit_params_status_to_result; + use crate::context::params::LlamaContextParams; use crate::error::fit_error::FitError; + use crate::error::llama_cpp_error::LlamaCppError; + use crate::error::model_params_error::ModelParamsError; + use crate::llama_backend::LlamaBackend; + use crate::max_devices::max_devices; use crate::model::llama_lazy_mode::LlamaLazyMode; use crate::model::llama_lazy_mode_parse_error::LlamaLazyModeParseError; use crate::model::llama_load_mode::LlamaLoadMode; + use crate::model::params::param_override_value::ParamOverrideValue; use crate::model::split_mode::LlamaSplitMode; - fn contract_error_reported_for( - status: llama_cpp_bindings_sys::llama_rs_fit_params_status, - ) -> Option { - fit_params_status_to_result(status, 0, ptr::null_mut()).err() - } - - fn contract_error_with(detail: &'static str) -> FitError { - crate::FfiContractError { - operation: "llama_rs_fit_params", - detail, - } - .into() - } - #[test] - fn device_capacity_is_not_smaller_than_the_upstream_limit() { - let upstream_limit = crate::max_devices(); - assert!( - LLAMA_CPP_MAX_DEVICES >= upstream_limit, - "LLAMA_CPP_MAX_DEVICES ({LLAMA_CPP_MAX_DEVICES}) backs a fixed-size device array and is \ - clamped against llama_max_devices() ({upstream_limit}); once upstream reports more \ - devices than the array can hold, the extra devices are dropped without any error" - ); - } - - #[test] - fn default_params_have_expected_values() { + fn default_params_match_the_llama_cpp_model_defaults() { let params = LlamaModelParams::default(); assert_eq!(params.n_gpu_layers(), -1); @@ -575,13 +536,6 @@ mod tests { assert_eq!(params.n_gpu_layers(), 999); } - #[test] - fn with_n_gpu_layers_sets_value() { - let params = LlamaModelParams::default().with_n_gpu_layers(32); - - assert_eq!(params.n_gpu_layers(), 32); - } - #[test] fn with_main_gpu_sets_value() { let params = LlamaModelParams::default().with_main_gpu(2); @@ -590,19 +544,12 @@ mod tests { } #[test] - fn with_split_mode_none() { + fn with_split_mode_none_disables_splitting() { let params = LlamaModelParams::default().with_split_mode(LlamaSplitMode::None); assert_eq!(params.split_mode(), Ok(LlamaSplitMode::None)); } - #[test] - fn with_split_mode_row() { - let params = LlamaModelParams::default().with_split_mode(LlamaSplitMode::Row); - - assert_eq!(params.split_mode(), Ok(LlamaSplitMode::Row)); - } - #[test] fn with_vocab_only_enables() { let params = LlamaModelParams::default().with_vocab_only(true); @@ -611,8 +558,10 @@ mod tests { } #[test] - fn with_vocab_only_disables() { - let params = LlamaModelParams::default().with_vocab_only(false); + fn with_vocab_only_false_disables_vocab_only_loading() { + let params = LlamaModelParams::default() + .with_vocab_only(true) + .with_vocab_only(false); assert!(!params.vocab_only()); } @@ -625,8 +574,10 @@ mod tests { } #[test] - fn with_no_alloc_disables() { - let params = LlamaModelParams::default().with_no_alloc(false); + fn with_no_alloc_false_reenables_allocation() { + let params = LlamaModelParams::default() + .with_no_alloc(true) + .with_no_alloc(false); assert!(!params.no_alloc()); } @@ -649,8 +600,10 @@ mod tests { } #[test] - fn with_load_mtp_disables_mtp_loading() { - let params = LlamaModelParams::default().with_load_mtp(false); + fn with_load_mtp_false_disables_mtp_loading() { + let params = LlamaModelParams::default() + .with_load_mtp(true) + .with_load_mtp(false); assert!(!params.load_mtp()); } @@ -663,23 +616,16 @@ mod tests { } #[test] - fn debug_format_contains_field_names() { - let params = LlamaModelParams::default(); - let debug_output = format!("{params:?}"); - - assert!(debug_output.contains("n_gpu_layers")); - assert!(debug_output.contains("main_gpu")); - assert!(debug_output.contains("vocab_only")); - assert!(debug_output.contains("load_mode")); - assert!(debug_output.contains("split_mode")); - } - - #[test] - fn debug_format_includes_lazy_mode() { - let debug_output = format!("{:?}", LlamaModelParams::default()); + fn debug_output_lists_every_field_including_the_lazy_mode() { + let params = LlamaModelParams::default().with_lazy_mode(LlamaLazyMode::On); - assert!(debug_output.contains("lazy_mode")); - assert!(debug_output.contains("Auto")); + assert_eq!( + format!("{params:?}"), + "LlamaModelParams { n_gpu_layers: -1, main_gpu: 0, vocab_only: false, \ + lazy_mode: Ok(On), load_mode: Ok(Auto), load_mtp: false, \ + split_mode: Ok(Layer), devices: [], \ + kv_overrides: \"vec of kv_overrides\", .. }" + ); } #[test] @@ -699,60 +645,83 @@ mod tests { } #[test] - fn with_devices_empty_list_succeeds() { - let params = LlamaModelParams::default().with_devices(&[]); - - assert!(params.is_ok()); - assert_eq!(params.unwrap().devices(), Vec::::new()); + fn with_devices_accepts_an_empty_device_list() { + assert_eq!( + LlamaModelParams::default() + .with_devices(&[]) + .map(|params| params.devices()), + Ok(Vec::new()) + ); } #[test] - fn with_devices_invalid_index_returns_error() { - let result = LlamaModelParams::default().with_devices(&[999_999]); + fn add_cpu_buft_override_fills_an_empty_override_slot() { + let mut params = pin!(LlamaModelParams::default()); assert_eq!( - std::mem::discriminant(&result.unwrap_err()), - std::mem::discriminant(&crate::LlamaCppError::BackendDeviceNotFound(0)), + params.as_mut().add_cpu_buft_override(c"test_pattern"), + Ok(()) + ); + assert_eq!( + unsafe { CStr::from_ptr(params.buft_overrides[0].pattern) }, + c"test_pattern" + ); + assert_eq!(params.buft_overrides[0].buft, unsafe { + llama_cpp_bindings_sys::ggml_backend_cpu_buffer_type() + }); + assert_eq!(params.buft_overrides[1].pattern, ptr::null()); + assert_eq!( + params.params.tensor_buft_overrides, + params.buft_overrides.as_ptr() ); } #[test] - fn add_cpu_buft_override_succeeds() { - let mut params = std::pin::pin!(LlamaModelParams::default()); - let result = params.as_mut().add_cpu_buft_override(c"test_pattern"); + fn a_buft_override_pattern_stays_valid_after_the_callers_buffer_changes() { + let mut caller_buffer = b"blk\\.0\\.ffn\0".to_vec(); + let mut params = pin!(LlamaModelParams::default()); - assert!(result.is_ok()); + params + .as_mut() + .add_cpu_buft_override(CStr::from_bytes_with_nul(&caller_buffer).unwrap()) + .unwrap(); + caller_buffer.fill(b'x'); + + assert_eq!( + unsafe { CStr::from_ptr(params.buft_overrides[0].pattern) }, + c"blk\\.0\\.ffn" + ); } #[test] fn add_cpu_buft_override_twice_fails_with_slot_not_empty() { - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); params .as_mut() .add_cpu_buft_override(c"first_pattern") .unwrap(); let result = params.as_mut().add_cpu_buft_override(c"second_pattern"); - assert_eq!( - result.unwrap_err(), - crate::error::ModelParamsError::SlotNotEmpty - ); + assert_eq!(result.unwrap_err(), ModelParamsError::SlotNotEmpty); } #[test] - fn add_cpu_moe_override_succeeds() { - let mut params = std::pin::pin!(LlamaModelParams::default()); - let result = params.as_mut().add_cpu_moe_override(); + fn add_cpu_moe_override_keeps_expert_tensors_in_cpu_memory() { + let mut params = pin!(LlamaModelParams::default()); - assert!(result.is_ok()); + assert_eq!(params.as_mut().add_cpu_moe_override(), Ok(())); + assert_eq!( + unsafe { CStr::from_ptr(params.buft_overrides[0].pattern) }, + c"\\.ffn_(up|down|gate)_(ch|)exps" + ); + assert_eq!(params.buft_overrides[0].buft, unsafe { + llama_cpp_bindings_sys::ggml_backend_cpu_buffer_type() + }); } #[test] fn append_kv_override_twice_fails_with_slot_not_empty() { - use crate::model::params::param_override_value::ParamOverrideValue; - use std::ffi::CString; - - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); let key = CString::new("first_key").unwrap(); params .as_mut() @@ -764,37 +733,74 @@ mod tests { .as_mut() .append_kv_override(&key2, ParamOverrideValue::Int(2)); + assert_eq!(result.unwrap_err(), ModelParamsError::SlotNotEmpty); + } + + #[test] + fn more_devices_than_llama_cpp_supports_are_refused() { + let too_many = vec![0; max_devices() + 1]; + assert_eq!( - result.unwrap_err(), - crate::error::ModelParamsError::SlotNotEmpty + LlamaModelParams::default().with_devices(&too_many).err(), + Some(LlamaCppError::MaxDevicesExceeded(max_devices())) ); } #[test] - fn with_devices_too_many_returns_max_exceeded() { - let too_many: Vec = (0..17).collect(); - let result = LlamaModelParams::default().with_devices(&too_many); + fn a_full_device_list_stays_null_terminated() { + let params = LlamaModelParams::default() + .with_devices(&vec![0; max_devices()]) + .unwrap(); + + assert_eq!(params.devices.len(), max_devices() + 1); + assert_eq!(params.devices.last(), Some(&ptr::null_mut())); + } + + #[test] + fn a_kv_override_key_of_any_bytes_is_stored_as_given() { + let key = c"cl\xc3\xa9"; + let mut params = pin!(LlamaModelParams::default()); - assert!( - result - .unwrap_err() - .to_string() - .contains("Max devices exceeded") + params + .as_mut() + .append_kv_override(key, ParamOverrideValue::Int(1)) + .unwrap(); + + assert_eq!( + unsafe { CStr::from_ptr(params.kv_overrides[0].key.as_ptr()) }, + key + ); + } + + #[test] + fn a_kv_override_key_longer_than_llama_cpp_holds_is_refused() { + let key = CString::new("k".repeat(KV_OVERRIDE_TEXT_CAPACITY)).unwrap(); + let mut params = pin!(LlamaModelParams::default()); + + assert_eq!( + params + .as_mut() + .append_kv_override(&key, ParamOverrideValue::Int(1)), + Err(ModelParamsError::KeyTooLong { + length_with_nul: KV_OVERRIDE_TEXT_CAPACITY + 1, + capacity: KV_OVERRIDE_TEXT_CAPACITY, + }) ); } #[test] fn with_devices_sets_devices_when_available() { #[cfg(feature = "dynamic-backends")] - crate::load_backends::load_backends().unwrap(); + unsafe { + llama_cpp_bindings_sys::ggml_backend_load_all(); + } let dev_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; assert!(dev_count > 0, "Test requires at least one backend device"); let params = LlamaModelParams::default().with_devices(&[0]).unwrap(); - assert_eq!(params.devices().len(), 1); - assert_eq!(params.devices()[0], 0); + assert_eq!(params.devices(), vec![0]); } #[test] @@ -802,88 +808,46 @@ mod tests { let invalid_index = usize::MAX; let result = LlamaModelParams::default().with_devices(&[invalid_index]); - assert!(result.unwrap_err().to_string().contains("Backend device")); - } - - #[test] - fn append_kv_override_with_high_byte_returns_invalid_character_error() { - use crate::model::params::param_override_value::ParamOverrideValue; - - let key_bytes: &[u8] = b"\xff\0"; - let key = std::ffi::CStr::from_bytes_with_nul(key_bytes).unwrap(); - let mut params = std::pin::pin!(LlamaModelParams::default()); - let result = params - .as_mut() - .append_kv_override(key, ParamOverrideValue::Int(1)); - - assert_eq!( - std::mem::discriminant(&result.unwrap_err()), - std::mem::discriminant(&crate::error::ModelParamsError::InvalidCharacterInKey { - byte: 0, - reason: String::new(), - }), - ); - } - - #[test] - fn add_cpu_buft_override_with_high_byte_returns_invalid_character_error() { - let key_bytes: &[u8] = b"\xff\0"; - let key = std::ffi::CStr::from_bytes_with_nul(key_bytes).unwrap(); - let mut params = std::pin::pin!(LlamaModelParams::default()); - let result = params.as_mut().add_cpu_buft_override(key); - - assert_eq!( - std::mem::discriminant(&result.unwrap_err()), - std::mem::discriminant(&crate::error::ModelParamsError::InvalidCharacterInKey { - byte: 0, - reason: String::new(), - }), + assert!( + matches!( + result, + Err(LlamaCppError::BackendDeviceNotFound(device)) if device == invalid_index + ), + "an out-of-range device index must not be found, got {result:?}" ); } #[test] fn append_kv_override_with_empty_slot_vector_returns_no_available_slot() { - use crate::model::params::param_override_value::ParamOverrideValue; - let mut params = LlamaModelParams::default(); params.kv_overrides.clear(); - let mut pinned = std::pin::pin!(params); + let mut pinned = pin!(params); let result = pinned .as_mut() .append_kv_override(c"any_key", ParamOverrideValue::Int(1)); - assert_eq!( - result.unwrap_err(), - crate::error::ModelParamsError::NoAvailableSlot - ); + assert_eq!(result.unwrap_err(), ModelParamsError::NoAvailableSlot); } #[test] fn add_cpu_buft_override_with_empty_slot_vector_returns_no_available_slot() { let mut params = LlamaModelParams::default(); params.buft_overrides.clear(); - let mut pinned = std::pin::pin!(params); + let mut pinned = pin!(params); let result = pinned.as_mut().add_cpu_buft_override(c"any_pattern"); - assert_eq!( - result.unwrap_err(), - crate::error::ModelParamsError::NoAvailableSlot - ); + assert_eq!(result.unwrap_err(), ModelParamsError::NoAvailableSlot); } #[test] #[serial_test::serial] - fn fit_params_invalid_model_path_returns_error() { - use crate::context::params::LlamaContextParams; - use crate::error::FitError; - use crate::llama_backend::LlamaBackend; - + fn fit_params_reports_a_missing_model_file_as_aborted() { let _backend = LlamaBackend::init(); - let mut params = std::pin::pin!(LlamaModelParams::default()); + let mut params = pin!(LlamaModelParams::default()); let mut context_params = LlamaContextParams::default(); - let mut margins = vec![0usize; crate::max_devices()]; + let mut margins = vec![0usize; max_devices()]; let bogus_path = c"/nonexistent/path/to/model.gguf"; let result = params.as_mut().fit_params( @@ -895,108 +859,59 @@ mod tests { llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, ); - assert!( - matches!(result, Err(FitError::Aborted | FitError::Reported { .. })), - "expected Aborted or Reported, got {result:?}" - ); + assert_eq!(result.err(), Some(FitError::Aborted)); } #[test] - fn fit_params_status_to_result_maps_null_path_model_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG - ), - Some(contract_error_with("was given a null path_model argument")) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_mparams_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG - ), - Some(contract_error_with("was given a null mparams argument")) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_cparams_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG - ), - Some(contract_error_with("was given a null cparams argument")) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_out_unrecognized_status_code_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_UNRECOGNIZED_STATUS_CODE_ARG - ), - Some(contract_error_with( - "was given a null out_unrecognized_status_code argument" - )) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_out_error_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG - ), - Some(contract_error_with("was given a null out_error argument")) - ); - } - - #[test] - fn fit_params_status_to_result_maps_null_extra_path_model_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG - ), - Some(contract_error_with( - "was given an extra model with a null path_model argument" - )) - ); - } + fn fit_params_refuses_margins_not_sized_to_the_device_count() { + let mut params = pin!(LlamaModelParams::default()); + let mut context_params = LlamaContextParams::default(); + let mut margins = vec![0usize; max_devices() + 1]; - #[test] - fn fit_params_status_to_result_maps_null_extra_mparams_arg() { - assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG - ), - Some(contract_error_with( - "was given an extra model with a null mparams argument" - )) + let result = params.as_mut().fit_params( + c"/nonexistent/path/to/model.gguf", + &mut context_params, + &mut margins, + 512, + None, + llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE, ); - } - #[test] - fn fit_params_status_to_result_maps_null_extra_cparams_arg() { assert_eq!( - contract_error_reported_for( - llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG - ), - Some(contract_error_with( - "was given an extra model with a null cparams argument" - )) + result.err(), + Some(FitError::MarginsDeviceCountMismatch { + expected: max_devices(), + actual: max_devices() + 1, + }) ); } } #[cfg(test)] mod fit_params_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::fit_params_status_to_result; use crate::error::fit_error::FitError; + fn contract_error_reported_for( + status: llama_cpp_bindings_sys::llama_rs_fit_params_status, + ) -> Option { + fit_params_status_to_result(status, 0, ptr::null_mut()).err() + } + + fn contract_error_with(detail: &'static str) -> FitError { + FfiContractError { + operation: "llama_rs_fit_params", + detail, + } + .into() + } + #[test] fn fit_params_status_ok_returns_ok() { let result = fit_params_status_to_result( @@ -1053,7 +968,25 @@ mod fit_params_status_to_result_tests { } #[test] - fn fit_params_status_cxx_exception_without_a_message_is_a_contract_error_with_unknown_error() { + fn fit_params_status_cxx_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"fitting threw".as_ptr()) }; + let result = fit_params_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ); + + assert_eq!( + result, + Err(FitError::Reported { + message: "fitting threw".to_owned() + }) + ); + } + + #[test] + fn fit_params_status_reports_an_exception_without_a_message_as_a_contract_violation() { let result = fit_params_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_LLAMA_CPP_THREW_CXX_EXCEPTION, 0, @@ -1062,10 +995,10 @@ mod fit_params_status_to_result_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_fit_params", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -1076,7 +1009,7 @@ mod fit_params_status_to_result_tests { assert_eq!( result, - Err(FitError::FfiStatus(crate::FfiStatusError { + Err(FitError::FfiStatus(FfiStatusError { operation: "llama_rs_fit_params", code: 255, })) @@ -1093,4 +1026,93 @@ mod fit_params_status_to_result_tests { assert_eq!(outcome.err(), Some(FitError::LlamaCppOutOfMemory)); } + + #[test] + fn fit_params_status_to_result_reports_a_null_path_model_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_PATH_MODEL_ARG + ), + Some(contract_error_with("was given a null path_model argument")) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_mparams_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_MPARAMS_ARG + ), + Some(contract_error_with("was given a null mparams argument")) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_cparams_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_CPARAMS_ARG + ), + Some(contract_error_with("was given a null cparams argument")) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_out_unrecognized_status_code_arg_as_a_contract_violation() + { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_UNRECOGNIZED_STATUS_CODE_ARG + ), + Some(contract_error_with( + "was given a null out_unrecognized_status_code argument" + )) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_OUT_ERROR_ARG + ), + Some(contract_error_with("was given a null out_error argument")) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_extra_path_model_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_PATH_MODEL_ARG + ), + Some(contract_error_with( + "was given an extra model with a null path_model argument" + )) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_extra_mparams_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_MPARAMS_ARG + ), + Some(contract_error_with( + "was given an extra model with a null mparams argument" + )) + ); + } + + #[test] + fn fit_params_status_to_result_reports_a_null_extra_cparams_arg_as_a_contract_violation() { + assert_eq!( + contract_error_reported_for( + llama_cpp_bindings_sys::LLAMA_RS_FIT_PARAMS_NULL_EXTRA_CPARAMS_ARG + ), + Some(contract_error_with( + "was given an extra model with a null cparams argument" + )) + ); + } } diff --git a/llama-cpp-bindings/src/model/params/empty_kv_override.rs b/llama-cpp-bindings/src/model/params/empty_kv_override.rs new file mode 100644 index 000000000..92f68fefe --- /dev/null +++ b/llama-cpp-bindings/src/model/params/empty_kv_override.rs @@ -0,0 +1,9 @@ +use std::mem; + +/// A KV override with every byte zero, which llama.cpp reads as the end of an override list. +pub const EMPTY_KV_OVERRIDE: llama_cpp_bindings_sys::llama_model_kv_override = + unsafe { mem::zeroed() }; + +/// How many bytes, its NUL included, a string KV override value holds in llama.cpp. +pub const KV_OVERRIDE_TEXT_CAPACITY: usize = + unsafe { EMPTY_KV_OVERRIDE.__bindgen_anon_1.val_str }.len(); diff --git a/llama-cpp-bindings/src/model/params/kv_overrides.rs b/llama-cpp-bindings/src/model/params/kv_overrides.rs index 822e27106..b27ea6083 100644 --- a/llama-cpp-bindings/src/model/params/kv_overrides.rs +++ b/llama-cpp-bindings/src/model/params/kv_overrides.rs @@ -26,12 +26,13 @@ impl<'model_params> IntoIterator for KvOverrides<'model_params> { #[cfg(test)] mod tests { - use crate::model::params::kv_override_entry::KvOverrideEntry; use std::ffi::CString; use std::pin::pin; use crate::model::params::LlamaModelParams; + use crate::model::params::kv_override_entry::KvOverrideEntry; use crate::model::params::param_override_value::ParamOverrideValue; + use crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag; #[test] fn kv_overrides_empty_by_default() { @@ -64,15 +65,6 @@ mod tests { ); } - #[test] - fn kv_overrides_new_creates_view() { - let params = LlamaModelParams::default(); - let overrides = super::KvOverrides::new(¶ms); - let count = overrides.into_iter().count(); - - assert_eq!(count, 0); - } - #[test] fn kv_overrides_preserves_unknown_tag_error() { let mut params = pin!(LlamaModelParams::default()); @@ -91,9 +83,6 @@ mod tests { .next() .expect("one override must be present"); - assert_eq!( - entry.unwrap_err(), - crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag(9999) - ); + assert_eq!(entry.unwrap_err(), UnknownKvOverrideTag(9999)); } } diff --git a/llama-cpp-bindings/src/model/params/param_override_value.rs b/llama-cpp-bindings/src/model/params/param_override_value.rs index 041371a54..c7a47f627 100644 --- a/llama-cpp-bindings/src/model/params/param_override_value.rs +++ b/llama-cpp-bindings/src/model/params/param_override_value.rs @@ -1,3 +1,6 @@ +use std::ffi::c_char; + +use crate::model::params::empty_kv_override::KV_OVERRIDE_TEXT_CAPACITY; use crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag; #[derive(Debug, Clone, Copy, PartialEq)] @@ -5,7 +8,7 @@ pub enum ParamOverrideValue { Bool(bool), Float(f64), Int(i64), - Str([std::os::raw::c_char; 128]), + Str([c_char; KV_OVERRIDE_TEXT_CAPACITY]), } impl ParamOverrideValue { @@ -68,50 +71,34 @@ impl TryFrom<&llama_cpp_bindings_sys::llama_model_kv_override> for ParamOverride #[cfg(test)] mod tests { - use super::ParamOverrideValue; - - #[test] - fn tag_bool() { - let value = ParamOverrideValue::Bool(true); - - assert_eq!( - value.tag(), - llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_BOOL - ); - } - - #[test] - fn tag_float() { - let value = ParamOverrideValue::Float(1.23); - - assert_eq!( - value.tag(), - llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_FLOAT - ); - } + use std::mem; - #[test] - fn tag_int() { - let value = ParamOverrideValue::Int(42); - - assert_eq!( - value.tag(), - llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_INT - ); - } + use super::ParamOverrideValue; + use crate::model::params::empty_kv_override::EMPTY_KV_OVERRIDE; + use crate::model::params::empty_kv_override::KV_OVERRIDE_TEXT_CAPACITY; + use crate::model::params::unknown_kv_override_tag::UnknownKvOverrideTag; #[test] - fn tag_str() { - let value = ParamOverrideValue::Str([0; 128]); - + fn each_override_value_is_tagged_with_its_llama_cpp_type() { assert_eq!( - value.tag(), - llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_STR + [ + ParamOverrideValue::Bool(true), + ParamOverrideValue::Float(1.23), + ParamOverrideValue::Int(42), + ParamOverrideValue::Str([0; KV_OVERRIDE_TEXT_CAPACITY]), + ] + .map(|value| value.tag()), + [ + llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_BOOL, + llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_FLOAT, + llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_INT, + llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_STR, + ] ); } #[test] - fn value_bool_roundtrip() { + fn a_bool_override_stores_its_value_in_the_bool_field() { let value = ParamOverrideValue::Bool(true); let ffi_value = value.value(); let result = unsafe { ffi_value.val_bool }; @@ -120,7 +107,7 @@ mod tests { } #[test] - fn value_float_roundtrip() { + fn a_float_override_stores_its_value_in_the_f64_field() { let value = ParamOverrideValue::Float(1.23); let ffi_value = value.value(); let result = unsafe { ffi_value.val_f64 }; @@ -129,7 +116,7 @@ mod tests { } #[test] - fn value_int_roundtrip() { + fn an_int_override_stores_its_value_in_the_i64_field() { let value = ParamOverrideValue::Int(99); let ffi_value = value.value(); let result = unsafe { ffi_value.val_i64 }; @@ -138,13 +125,13 @@ mod tests { } #[test] - fn from_ffi_override_int() { + fn an_int_kv_override_reads_as_an_int_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_INT, __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { val_i64: 123, }, + ..EMPTY_KV_OVERRIDE }; let value = ParamOverrideValue::try_from(&ffi_override).unwrap(); @@ -153,13 +140,13 @@ mod tests { } #[test] - fn from_ffi_override_float() { + fn a_float_kv_override_reads_as_a_float_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_FLOAT, __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { val_f64: 1.5, }, + ..EMPTY_KV_OVERRIDE }; let value = ParamOverrideValue::try_from(&ffi_override).unwrap(); @@ -168,13 +155,13 @@ mod tests { } #[test] - fn from_ffi_override_bool() { + fn a_bool_kv_override_reads_as_a_bool_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_BOOL, __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { val_bool: false, }, + ..EMPTY_KV_OVERRIDE }; let value = ParamOverrideValue::try_from(&ffi_override).unwrap(); @@ -183,8 +170,8 @@ mod tests { } #[test] - fn value_str_roundtrip() { - let mut str_data = [0i8; 128]; + fn a_str_override_stores_its_bytes_in_the_str_field() { + let mut str_data = [0; KV_OVERRIDE_TEXT_CAPACITY]; str_data[0] = b'h'.cast_signed(); str_data[1] = b'i'.cast_signed(); @@ -197,17 +184,17 @@ mod tests { } #[test] - fn from_ffi_override_str() { - let mut str_data = [0i8; 128]; + fn a_str_kv_override_reads_as_a_str_value() { + let mut str_data = [0; KV_OVERRIDE_TEXT_CAPACITY]; str_data[0] = b'a'.cast_signed(); str_data[1] = b'b'.cast_signed(); let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: llama_cpp_bindings_sys::LLAMA_KV_OVERRIDE_TYPE_STR, __bindgen_anon_1: llama_cpp_bindings_sys::llama_model_kv_override__bindgen_ty_1 { val_str: str_data, }, + ..EMPTY_KV_OVERRIDE }; let value = ParamOverrideValue::try_from(&ffi_override).unwrap(); @@ -216,15 +203,16 @@ mod tests { } #[test] - fn unknown_tag_returns_error() { + fn an_unknown_kv_override_tag_is_reported_with_its_value() { let ffi_override = llama_cpp_bindings_sys::llama_model_kv_override { - key: [0; 128], tag: 9999, - __bindgen_anon_1: unsafe { std::mem::zeroed() }, + __bindgen_anon_1: unsafe { mem::zeroed() }, + ..EMPTY_KV_OVERRIDE }; - let result = ParamOverrideValue::try_from(&ffi_override); - - assert!(result.is_err()); + assert_eq!( + ParamOverrideValue::try_from(&ffi_override), + Err(UnknownKvOverrideTag(9999)) + ); } } diff --git a/llama-cpp-bindings/src/model/piece_write.rs b/llama-cpp-bindings/src/model/piece_write.rs new file mode 100644 index 000000000..3aea97eee --- /dev/null +++ b/llama-cpp-bindings/src/model/piece_write.rs @@ -0,0 +1,36 @@ +use std::ffi::c_int; + +/// What `llama_token_to_piece` did with the buffer it was given: wrote the piece's bytes, or +/// reported, as a negative size, the bytes the piece needs. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PieceWrite { + Written(usize), + NeedsBuffer(c_int), +} + +impl From for PieceWrite { + fn from(size: c_int) -> Self { + if size.is_negative() { + Self::NeedsBuffer(size) + } else { + Self::Written(size.unsigned_abs() as usize) + } + } +} + +#[cfg(test)] +mod tests { + use super::PieceWrite; + + #[test] + fn a_negative_size_asks_for_a_larger_buffer_and_any_other_is_the_written_length() { + assert_eq!( + [-7, 0, 5].map(PieceWrite::from), + [ + PieceWrite::NeedsBuffer(-7), + PieceWrite::Written(0), + PieceWrite::Written(5) + ] + ); + } +} diff --git a/llama-cpp-bindings/src/model/rope_type.rs b/llama-cpp-bindings/src/model/rope_type.rs index cf3173db6..b3d75de40 100644 --- a/llama-cpp-bindings/src/model/rope_type.rs +++ b/llama-cpp-bindings/src/model/rope_type.rs @@ -1,70 +1,89 @@ +use crate::error::unknown_rope_type::UnknownRopeType; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RopeType { + None, Norm, NeoX, MRope, + IMRope, Vision, } +impl TryFrom for RopeType { + type Error = UnknownRopeType; + + fn try_from(value: llama_cpp_bindings_sys::llama_rope_type) -> Result { + match value { + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NONE => Ok(Self::None), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM => Ok(Self::Norm), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX => Ok(Self::NeoX), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE => Ok(Self::MRope), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE => Ok(Self::IMRope), + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION => Ok(Self::Vision), + unknown => Err(UnknownRopeType(unknown)), + } + } +} + impl RopeType { + /// Whether every embedding carries a single position, which llama.cpp requires before it + /// shifts or divides the positions of a sequence in memory. #[must_use] - pub const fn from_raw(raw: i32) -> Option { - match raw { - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM => Some(Self::Norm), - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX => Some(Self::NeoX), - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE => Some(Self::MRope), - llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION => Some(Self::Vision), - _ => None, + pub const fn has_one_position_per_embedding(self) -> bool { + match self { + Self::None | Self::Norm | Self::NeoX | Self::Vision => true, + Self::MRope | Self::IMRope => false, } } } #[cfg(test)] mod tests { - use super::RopeType; + use crate::error::unknown_rope_type::UnknownRopeType; + use crate::model::rope_type::RopeType; #[test] - fn rope_type_none() { + fn every_llama_cpp_rope_type_converts_to_its_variant() { assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NONE), - None + [ + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NONE, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_IMROPE, + llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION, + ] + .map(RopeType::try_from), + [ + Ok(RopeType::None), + Ok(RopeType::Norm), + Ok(RopeType::NeoX), + Ok(RopeType::MRope), + Ok(RopeType::IMRope), + Ok(RopeType::Vision), + ] ); } #[test] - fn rope_type_norm() { - assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NORM), - Some(RopeType::Norm) - ); + fn a_value_llama_cpp_does_not_define_is_refused() { + assert_eq!(RopeType::try_from(9999), Err(UnknownRopeType(9999))); } #[test] - fn rope_type_neox() { + fn only_multi_section_rope_types_place_several_positions_per_embedding() { assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_NEOX), - Some(RopeType::NeoX) + [ + RopeType::None, + RopeType::Norm, + RopeType::NeoX, + RopeType::MRope, + RopeType::IMRope, + RopeType::Vision, + ] + .map(RopeType::has_one_position_per_embedding), + [true, true, true, false, false, true] ); } - - #[test] - fn rope_type_mrope() { - assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_MROPE), - Some(RopeType::MRope) - ); - } - - #[test] - fn rope_type_vision() { - assert_eq!( - RopeType::from_raw(llama_cpp_bindings_sys::LLAMA_ROPE_TYPE_VISION), - Some(RopeType::Vision) - ); - } - - #[test] - fn rope_type_unknown_returns_none() { - assert_eq!(RopeType::from_raw(9999), None); - } } diff --git a/llama-cpp-bindings/src/model/split_mode.rs b/llama-cpp-bindings/src/model/split_mode.rs index e1c359df9..bae6020a2 100644 --- a/llama-cpp-bindings/src/model/split_mode.rs +++ b/llama-cpp-bindings/src/model/split_mode.rs @@ -42,58 +42,41 @@ impl From for llama_cpp_bindings_sys::llama_split_mode { #[cfg(test)] mod tests { use super::LlamaSplitMode; + use crate::model::llama_split_mode_parse_error::LlamaSplitModeParseError; #[test] fn try_from_invalid_reports_the_value() { - let result = LlamaSplitMode::try_from(99); - - assert!(result.is_err()); - assert_eq!(result.unwrap_err().value, 99); - } - - #[test] - fn try_from_none_roundtrip() { - let mode = LlamaSplitMode::try_from(llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_NONE).unwrap(); - - assert_eq!(mode, LlamaSplitMode::None); assert_eq!( - llama_cpp_bindings_sys::llama_split_mode::from(mode), - llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_NONE + LlamaSplitMode::try_from(99), + Err(LlamaSplitModeParseError { + value: 99, + context: "unknown split mode value: 99".to_owned(), + }) ); } #[test] - fn try_from_layer_roundtrip() { - let mode = - LlamaSplitMode::try_from(llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_LAYER).unwrap(); + fn every_llama_cpp_split_mode_converts_to_its_variant_and_back() { + let llama_cpp_split_modes = [ + llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_NONE, + llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_LAYER, + llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_ROW, + llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_TENSOR, + ]; + let modes = llama_cpp_split_modes.map(LlamaSplitMode::try_from); - assert_eq!(mode, LlamaSplitMode::Layer); assert_eq!( - llama_cpp_bindings_sys::llama_split_mode::from(mode), - llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_LAYER + modes, + [ + Ok(LlamaSplitMode::None), + Ok(LlamaSplitMode::Layer), + Ok(LlamaSplitMode::Row), + Ok(LlamaSplitMode::Tensor), + ] ); - } - - #[test] - fn try_from_row_roundtrip() { - let mode = LlamaSplitMode::try_from(llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_ROW).unwrap(); - - assert_eq!(mode, LlamaSplitMode::Row); - assert_eq!( - llama_cpp_bindings_sys::llama_split_mode::from(mode), - llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_ROW - ); - } - - #[test] - fn try_from_tensor_roundtrip() { - let mode = - LlamaSplitMode::try_from(llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_TENSOR).unwrap(); - - assert_eq!(mode, LlamaSplitMode::Tensor); assert_eq!( - llama_cpp_bindings_sys::llama_split_mode::from(mode), - llama_cpp_bindings_sys::LLAMA_SPLIT_MODE_TENSOR + modes.map(|mode| mode.map(llama_cpp_bindings_sys::llama_split_mode::from)), + llama_cpp_split_modes.map(Ok) ); } diff --git a/llama-cpp-bindings/src/model/tokenizer_input.rs b/llama-cpp-bindings/src/model/tokenizer_input.rs index 7953a796c..8e0b23d49 100644 --- a/llama-cpp-bindings/src/model/tokenizer_input.rs +++ b/llama-cpp-bindings/src/model/tokenizer_input.rs @@ -1,5 +1,5 @@ use std::ffi::CString; -use std::os::raw::c_int; +use std::ffi::c_int; #[derive(Debug, Eq, PartialEq)] pub struct TokenizerInput { diff --git a/llama-cpp-bindings/src/model/vocab_type.rs b/llama-cpp-bindings/src/model/vocab_type.rs index 14e151322..031ec7fd8 100644 --- a/llama-cpp-bindings/src/model/vocab_type.rs +++ b/llama-cpp-bindings/src/model/vocab_type.rs @@ -3,8 +3,15 @@ use crate::model::vocab_type_from_int_error::VocabTypeFromIntError; #[repr(u32)] #[derive(Debug, Eq, Copy, Clone, PartialEq)] pub enum VocabType { - BPE = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE as _, - SPM = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM as _, + None = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_NONE as _, + Spm = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM as _, + Bpe = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE as _, + Wpm = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_WPM as _, + Ugm = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_UGM as _, + Rwkv = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_RWKV as _, + Plamo2 = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO2 as _, + Test = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_TEST as _, + Plamo3 = llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO3 as _, } impl TryFrom for VocabType { @@ -12,8 +19,15 @@ impl TryFrom for VocabType { fn try_from(value: llama_cpp_bindings_sys::llama_vocab_type) -> Result { match value { - llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE => Ok(Self::BPE), - llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM => Ok(Self::SPM), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_NONE => Ok(Self::None), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM => Ok(Self::Spm), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE => Ok(Self::Bpe), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_WPM => Ok(Self::Wpm), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_UGM => Ok(Self::Ugm), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_RWKV => Ok(Self::Rwkv), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO2 => Ok(Self::Plamo2), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_TEST => Ok(Self::Test), + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO3 => Ok(Self::Plamo3), unknown => Err(VocabTypeFromIntError::UnknownValue(unknown)), } } @@ -21,24 +35,42 @@ impl TryFrom for VocabType { #[cfg(test)] mod tests { - use super::{VocabType, VocabTypeFromIntError}; + use crate::model::vocab_type::VocabType; + use crate::model::vocab_type_from_int_error::VocabTypeFromIntError; #[test] - fn try_from_bpe() { - let result = VocabType::try_from(llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE); + fn every_llama_cpp_vocab_type_maps_to_its_variant() { + let mapped = [ + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_NONE, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_BPE, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_WPM, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_UGM, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_RWKV, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO2, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_TEST, + llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_PLAMO3, + ] + .map(VocabType::try_from); - assert_eq!(result, Ok(VocabType::BPE)); + assert_eq!( + mapped, + [ + Ok(VocabType::None), + Ok(VocabType::Spm), + Ok(VocabType::Bpe), + Ok(VocabType::Wpm), + Ok(VocabType::Ugm), + Ok(VocabType::Rwkv), + Ok(VocabType::Plamo2), + Ok(VocabType::Test), + Ok(VocabType::Plamo3), + ] + ); } #[test] - fn try_from_spm() { - let result = VocabType::try_from(llama_cpp_bindings_sys::LLAMA_VOCAB_TYPE_SPM); - - assert_eq!(result, Ok(VocabType::SPM)); - } - - #[test] - fn try_from_unknown_value() { + fn a_vocab_type_unknown_to_llama_cpp_is_refused() { let result = VocabType::try_from(99999); assert_eq!(result, Err(VocabTypeFromIntError::UnknownValue(99999))); diff --git a/llama-cpp-bindings/src/model/vocabulary_token.rs b/llama-cpp-bindings/src/model/vocabulary_token.rs new file mode 100644 index 000000000..e81f98df6 --- /dev/null +++ b/llama-cpp-bindings/src/model/vocabulary_token.rs @@ -0,0 +1,8 @@ +use crate::token::LlamaToken; + +/// A token of the model's vocabulary with the text it decodes to. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VocabularyToken { + pub token: LlamaToken, + pub piece: String, +} diff --git a/llama-cpp-bindings/src/mtmd.rs b/llama-cpp-bindings/src/mtmd.rs index 2d7c4212e..f615b58a2 100644 --- a/llama-cpp-bindings/src/mtmd.rs +++ b/llama-cpp-bindings/src/mtmd.rs @@ -1,16 +1,21 @@ +mod eval_chunk; mod mtmd_ffi_input_part; +mod optional_mtmd_id; pub mod fitted_mtmd_input_chunk; pub mod fitted_mtmd_input_chunks; -pub mod micro_batch_tokens; pub mod mtmd_bitmap; pub mod mtmd_bitmap_error; +pub mod mtmd_capability; pub mod mtmd_context; pub mod mtmd_context_params; +pub mod mtmd_context_params_error; pub mod mtmd_default_marker; pub mod mtmd_default_marker_error; pub mod mtmd_encode_error; pub mod mtmd_eval_error; +pub mod mtmd_eval_limits; +pub mod mtmd_image_token_limit; pub mod mtmd_init_error; pub mod mtmd_input_chunk; pub mod mtmd_input_chunk_error; @@ -21,30 +26,7 @@ pub mod mtmd_input_chunks_error; pub mod mtmd_input_part; pub mod mtmd_input_text; pub mod mtmd_input_text_part; +pub mod mtmd_memory_usage; +pub mod mtmd_memory_usage_error; pub mod mtmd_tokenize_error; pub mod non_causal_chunk_micro_batch_mismatch; -pub mod positive_batch_tokens; - -pub use fitted_mtmd_input_chunk::FittedMtmdInputChunk; -pub use fitted_mtmd_input_chunks::FittedMtmdInputChunks; -pub use micro_batch_tokens::micro_batch_tokens; -pub use mtmd_bitmap::MtmdBitmap; -pub use mtmd_bitmap_error::MtmdBitmapError; -pub use mtmd_context::MtmdContext; -pub use mtmd_context_params::MtmdContextParams; -pub use mtmd_default_marker::mtmd_default_marker; -pub use mtmd_default_marker_error::MtmdDefaultMarkerError; -pub use mtmd_encode_error::MtmdEncodeError; -pub use mtmd_eval_error::MtmdEvalError; -pub use mtmd_init_error::MtmdInitError; -pub use mtmd_input_chunk::MtmdInputChunk; -pub use mtmd_input_chunk_error::MtmdInputChunkError; -pub use mtmd_input_chunk_type::MtmdInputChunkType; -pub use mtmd_input_chunk_type_error::MtmdInputChunkTypeError; -pub use mtmd_input_chunks::MtmdInputChunks; -pub use mtmd_input_chunks_error::MtmdInputChunksError; -pub use mtmd_input_part::MtmdInputPart; -pub use mtmd_input_text::MtmdInputText; -pub use mtmd_input_text_part::MtmdInputTextPart; -pub use mtmd_tokenize_error::MtmdTokenizeError; -pub use non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; diff --git a/llama-cpp-bindings/src/mtmd/eval_chunk.rs b/llama-cpp-bindings/src/mtmd/eval_chunk.rs new file mode 100644 index 000000000..1b1cb8163 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/eval_chunk.rs @@ -0,0 +1,315 @@ +use std::ffi::c_char; +use std::ptr; + +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + +use super::mtmd_context::MtmdContext; +use super::mtmd_eval_error::MtmdEvalError; +use super::mtmd_input_chunk::MtmdInputChunk; +use crate::context::LlamaContext; + +fn eval_chunk_single_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single_status, + final_position: llama_cpp_bindings_sys::llama_pos, + out_llama_cpp_return_code: i32, + out_error: *mut c_char, +) -> Result { + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK => Ok(final_position), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE => { + Err(MtmdEvalError::EvalFailed { + code: out_llama_cpp_return_code, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdEvalError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdEvalError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_mtmd_eval_chunk_single", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(MtmdEvalError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG => { + Err(FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null mtmd_ctx argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG => { + Err(FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null llama_ctx argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG => { + Err(FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null chunk argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG => { + Err(FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null out_new_n_past argument", + } + .into()) + } + other => Err(FfiStatusError { + operation: "llama_rs_mtmd_eval_chunk_single", + code: i64::from(other), + } + .into()), + } +} + +/// Encodes `chunk` when it holds media and decodes it into `llama_ctx` from `start_position`, +/// returning the position after it. +pub fn eval_chunk( + chunk: &MtmdInputChunk, + mtmd_ctx: &MtmdContext, + llama_ctx: &mut LlamaContext, + start_position: llama_cpp_bindings_sys::llama_pos, + seq_id: llama_cpp_bindings_sys::llama_seq_id, + n_batch: i32, + logits_last: bool, +) -> Result { + let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; + let mut out_llama_cpp_return_code: i32 = 0; + let mut out_error: *mut c_char = ptr::null_mut(); + + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single( + mtmd_ctx.context.as_ptr(), + llama_ctx.context.as_ptr(), + chunk.chunk.as_ptr(), + start_position, + seq_id, + n_batch, + logits_last, + &raw mut final_position, + &raw mut out_llama_cpp_return_code, + &raw mut out_error, + ) + }; + + let next_position = eval_chunk_single_status_to_result( + status, + final_position, + out_llama_cpp_return_code, + out_error, + )?; + llama_ctx.record_helper_decode(logits_last); + + Ok(next_position) +} + +#[cfg(test)] +mod eval_chunk_single_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::ptr; + + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; + + use super::eval_chunk_single_status_to_result; + use crate::mtmd::mtmd_eval_error::MtmdEvalError; + + #[test] + fn eval_chunk_single_status_cxx_exception_surfaces_the_reported_message() { + let expected_message = unsafe { + read_and_free_cpp_string( + wrapper_allocated_error_message_pointer(), + "llama_rs_mtmd_eval_chunk_single", + "reported a thrown C++ exception without an error message", + ) + }; + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + 0, + wrapper_allocated_error_message_pointer(), + ); + + assert_eq!( + result.err(), + expected_message + .ok() + .map(|message| MtmdEvalError::Reported { message }) + ); + } + + #[test] + fn eval_chunk_single_status_ok_returns_final_position() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK, + 7, + 0, + ptr::null_mut(), + ); + + assert_eq!(result, Ok(7)); + } + + #[test] + fn eval_chunk_single_status_nonzero_code_maps_to_eval_failed() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE, + 0, + -3, + ptr::null_mut(), + ); + + assert_eq!(result, Err(MtmdEvalError::EvalFailed { code: -3 })); + } + + #[test] + fn eval_chunk_single_status_allocation_failed_maps_to_not_enough_memory() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED, + 0, + 0, + ptr::null_mut(), + ); + + assert_eq!(result, Err(MtmdEvalError::NotEnoughMemory)); + } + + #[test] + fn eval_chunk_single_status_reports_an_exception_without_a_message_as_a_contract_violation() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + 0, + ptr::null_mut(), + ); + + assert_eq!( + result, + Err(CppStringError::Missing(FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "reported a thrown C++ exception without an error message", + }) + .into()) + ); + } + + #[test] + fn eval_chunk_single_unknown_status_is_preserved() { + let result = eval_chunk_single_status_to_result(255, 0, 0, ptr::null_mut()); + + assert_eq!( + result, + Err(MtmdEvalError::FfiStatus(FfiStatusError { + operation: "llama_rs_mtmd_eval_chunk_single", + code: 255, + })) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_reports_a_null_mtmd_ctx_arg_as_a_contract_violation() { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null mtmd_ctx argument", + } + .into() + ) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_reports_a_null_llama_ctx_arg_as_a_contract_violation() { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null llama_ctx argument", + } + .into() + ) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_reports_a_null_chunk_arg_as_a_contract_violation() { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null chunk argument", + } + .into() + ) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_reports_a_null_out_new_n_past_arg_as_a_contract_violation() + { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null out_new_n_past argument", + } + .into() + ) + ); + } + + #[test] + fn eval_chunk_single_status_to_result_reports_llama_cpp_running_out_of_memory() { + let outcome = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!(outcome.err(), Some(MtmdEvalError::LlamaCppOutOfMemory)); + } +} diff --git a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs index abdd1f4bf..b6237dbee 100644 --- a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs @@ -1,84 +1,15 @@ use std::borrow::Borrow; -use std::ffi::c_char; use std::marker::PhantomData; -use crate::context::LlamaContext; -use crate::token::LlamaToken; -use llama_cpp_ffi_status::read_and_free_cpp_string; - +use super::eval_chunk::eval_chunk; use super::fitted_mtmd_input_chunks::FittedMtmdInputChunks; -use super::micro_batch_tokens::micro_batch_tokens; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; +use super::mtmd_eval_limits::MtmdEvalLimits; use super::mtmd_input_chunk::MtmdInputChunk; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunks::MtmdInputChunks; -use super::positive_batch_tokens::positive_batch_tokens; - -fn eval_chunk_single_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single_status, - final_position: llama_cpp_bindings_sys::llama_pos, - out_llama_cpp_return_code: i32, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK => Ok(final_position), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE => { - Err(MtmdEvalError::EvalFailed { - code: out_llama_cpp_return_code, - }) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED => { - Err(MtmdEvalError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MtmdEvalError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_mtmd_eval_chunk_single", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(MtmdEvalError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null mtmd_ctx argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null llama_ctx argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null chunk argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null out_new_n_past argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_mtmd_eval_chunk_single", - code: i64::from(other), - } - .into()), - } -} +use crate::context::LlamaContext; #[derive(Debug)] pub struct FittedMtmdInputChunk<'fitted> { @@ -117,30 +48,25 @@ impl<'fitted> FittedMtmdInputChunk<'fitted> { self.chunk_type } - #[must_use] - pub fn text_tokens(&self) -> &[LlamaToken] { - self.chunk - .text_tokens_of(self.chunk_type) - .unwrap_or_default() - } - /// # Errors /// - /// Returns [`MtmdEvalError::NonPositiveBatchSize`] when `n_batch` is not positive, + /// Returns any error of [`MtmdEvalLimits::batch_tokens`], /// [`MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted`] when decodes of `n_batch` in /// `llama_ctx` are smaller than the micro batch the chunks were fitted to, or /// [`MtmdEvalError::EvalFailed`] if the underlying encode or decode step fails. pub fn eval_single( &self, mtmd_ctx: &MtmdContext, - llama_ctx: &LlamaContext, + llama_ctx: &mut LlamaContext, start_position: llama_cpp_bindings_sys::llama_pos, seq_id: llama_cpp_bindings_sys::llama_seq_id, n_batch: i32, logits_last: bool, ) -> Result { - let evaluated_micro_batch_tokens = - micro_batch_tokens(llama_ctx, positive_batch_tokens(n_batch)?); + let limits = MtmdEvalLimits::of(llama_ctx); + let evaluated_micro_batch_tokens = limits + .context + .micro_batch_tokens(limits.batch_tokens(n_batch, seq_id)?); if evaluated_micro_batch_tokens < self.micro_batch_tokens { return Err(MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted { @@ -149,224 +75,14 @@ impl<'fitted> FittedMtmdInputChunk<'fitted> { }); } - let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; - let mut out_llama_cpp_return_code: i32 = 0; - let mut out_error: *mut c_char = std::ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single( - mtmd_ctx.context.as_ptr(), - llama_ctx.context.as_ptr(), - self.chunk.chunk.as_ptr(), - start_position, - seq_id, - n_batch, - logits_last, - &raw mut final_position, - &raw mut out_llama_cpp_return_code, - &raw mut out_error, - ) - }; - - eval_chunk_single_status_to_result( - status, - final_position, - out_llama_cpp_return_code, - out_error, + eval_chunk( + &self.chunk, + mtmd_ctx, + llama_ctx, + start_position, + seq_id, + n_batch, + logits_last, ) } } - -#[cfg(test)] -mod eval_chunk_single_status_to_result_tests { - use std::ptr; - - use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; - - use super::eval_chunk_single_status_to_result; - use crate::mtmd::mtmd_eval_error::MtmdEvalError; - - #[test] - fn eval_chunk_single_status_cxx_exception_surfaces_the_reported_message() { - let expected_message = unsafe { - read_and_free_cpp_string( - wrapper_allocated_error_message_pointer(), - "llama_rs_mtmd_eval_chunk_single", - "reported a thrown C++ exception without an error message", - ) - }; - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - 0, - wrapper_allocated_error_message_pointer(), - ); - - assert_eq!( - result.err(), - expected_message - .ok() - .map(|message| MtmdEvalError::Reported { message }) - ); - } - - #[test] - fn eval_chunk_single_status_ok_returns_final_position() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK, - 7, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Ok(7)); - } - - #[test] - fn eval_chunk_single_status_nonzero_code_maps_to_eval_failed() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 0, - -3, - ptr::null_mut(), - ); - - assert_eq!(result, Err(MtmdEvalError::EvalFailed { code: -3 })); - } - - #[test] - fn eval_chunk_single_status_allocation_failed_maps_to_not_enough_memory() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED, - 0, - 0, - ptr::null_mut(), - ); - - assert_eq!(result, Err(MtmdEvalError::NotEnoughMemory)); - } - - #[test] - fn eval_chunk_single_status_cxx_exception_reports_unknown_error_for_null() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - 0, - ptr::null_mut(), - ); - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn eval_chunk_single_unknown_status_is_preserved() { - let result = eval_chunk_single_status_to_result(255, 0, 0, ptr::null_mut()); - - assert_eq!( - result, - Err(MtmdEvalError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_mtmd_eval_chunk_single", - code: 255, - })) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_null_mtmd_ctx_arg() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null mtmd_ctx argument", - } - .into() - ) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_null_llama_ctx_arg() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null llama_ctx argument", - } - .into() - ) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_null_chunk_arg() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null chunk argument", - } - .into() - ) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_null_out_new_n_past_arg() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null out_new_n_past argument", - } - .into() - ) - ); - } - - #[test] - fn eval_chunk_single_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!(outcome.err(), Some(MtmdEvalError::LlamaCppOutOfMemory)); - } -} diff --git a/llama-cpp-bindings/src/mtmd/micro_batch_tokens.rs b/llama-cpp-bindings/src/mtmd/micro_batch_tokens.rs deleted file mode 100644 index 3fb97599d..000000000 --- a/llama-cpp-bindings/src/mtmd/micro_batch_tokens.rs +++ /dev/null @@ -1,10 +0,0 @@ -use std::num::NonZeroU32; - -use crate::context::LlamaContext; - -/// Returns how many tokens a single decode of `n_batch` evaluates in `llama_ctx`: the smaller of -/// `n_batch` and the context's micro batch. -#[must_use] -pub fn micro_batch_tokens(llama_ctx: &LlamaContext, n_batch: NonZeroU32) -> u32 { - n_batch.get().min(llama_ctx.n_ubatch()) -} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs index 98d0d1271..0302807a1 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap.rs @@ -1,24 +1,18 @@ -use std::ffi::{CStr, CString, c_char}; +use std::ffi::NulError; +use std::ffi::{CString, c_char}; use std::path::PathBuf; +use std::ptr; use std::ptr::NonNull; use std::slice; +use std::str::Utf8Error; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::mtmd_bitmap_error::MtmdBitmapError; use super::mtmd_context::MtmdContext; - -fn cstr_ptr_to_optional_string(ptr: *const c_char) -> Option { - if ptr.is_null() { - None - } else { - let id = unsafe { CStr::from_ptr(ptr) } - .to_string_lossy() - .into_owned(); - - Some(id) - } -} +use super::optional_mtmd_id::optional_mtmd_id; /// # Safety /// @@ -35,7 +29,7 @@ unsafe fn from_file_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_OK => { let bitmap = NonNull::new(out_bitmap).ok_or_else(|| { - MtmdBitmapError::from(crate::FfiContractError { + MtmdBitmapError::from(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "success status contained a null bitmap", }) @@ -57,22 +51,22 @@ unsafe fn from_file_status_to_result( let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_mtmd_bitmap_init_from_file", "reported a thrown C++ exception without an error message") }?; Err(MtmdBitmapError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_CTX_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_CTX_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_FNAME_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_FNAME_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null fname argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_OUT_BITMAP_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_OUT_BITMAP_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null out_bitmap argument", } .into()), - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_bitmap_init_from_file", code: i64::from(other), } @@ -94,7 +88,7 @@ unsafe fn from_buf_status_to_result( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_OK => { let bitmap = NonNull::new(out_bitmap).ok_or_else(|| { - MtmdBitmapError::from(crate::FfiContractError { + MtmdBitmapError::from(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "success status contained a null bitmap", }) @@ -114,22 +108,22 @@ unsafe fn from_buf_status_to_result( let message = unsafe { read_and_free_cpp_string(out_error, "llama_rs_mtmd_bitmap_init_from_buf", "reported a thrown C++ exception without an error message") }?; Err(MtmdBitmapError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_CTX_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_CTX_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_BUF_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_BUF_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "was given a null buf argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_OUT_BITMAP_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_BUF_NULL_OUT_BITMAP_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "was given a null out_bitmap argument", } .into()), - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_bitmap_init_from_buf", code: i64::from(other), } @@ -137,19 +131,125 @@ unsafe fn from_buf_status_to_result( } } +fn bitmap_init_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_status, + out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_mtmd_bitmap_init"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_OK => NonNull::new(out_bitmap) + .map(|bitmap| MtmdBitmap { bitmap }) + .ok_or_else(|| { + FfiContractError { + operation: OPERATION, + detail: "success status contained a null bitmap", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG => Err(FfiContractError { + operation: OPERATION, + detail: "was given a null data argument", + } + .into()), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG => { + Err(FfiContractError { + operation: OPERATION, + detail: "was given a null out_bitmap argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdBitmapError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdBitmapError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdBitmapError::Reported { message }) + } + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +fn bitmap_init_from_audio_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_audio_status, + out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_mtmd_bitmap_init_from_audio"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK => NonNull::new(out_bitmap) + .map(|bitmap| MtmdBitmap { bitmap }) + .ok_or_else(|| { + FfiContractError { + operation: OPERATION, + detail: "success status contained a null bitmap", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG => Err(FfiContractError { + operation: OPERATION, + detail: "was given a null data argument", + } + .into()), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG => Err(FfiContractError { + operation: OPERATION, + detail: "was given a null out_bitmap argument", + } + .into()), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdBitmapError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdBitmapError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdBitmapError::Reported { message }) + } + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + /// Owns the bitmap and frees it on drop, so it cannot be cloned. /// /// ```compile_fail,E0277 /// fn require_clone() {} /// -/// require_clone::(); +/// require_clone::(); /// ``` #[derive(Debug)] pub struct MtmdBitmap { pub bitmap: NonNull, } -const RGB_BYTES_PER_PIXEL: usize = 3; +const RGB_BYTES_PER_PIXEL: u64 = 3; unsafe impl Send for MtmdBitmap {} unsafe impl Sync for MtmdBitmap {} @@ -160,42 +260,53 @@ impl MtmdBitmap { /// * `ImageDimensionsTooSmall` - `nx` or `ny` is below the 2x2 minimum /// * `ImageDimensionsOverflow` - `nx * ny * 3` does not fit into a buffer length /// * `InvalidDataSize` - Data length doesn't match `nx * ny * 3` - /// * `BitmapDecodeFailed` - Underlying C function returned null + /// * the [`MtmdBitmapError`] variant matching the wrapper's status code /// pub fn from_image_data(nx: u32, ny: u32, data: &[u8]) -> Result { if nx < 2 || ny < 2 { return Err(MtmdBitmapError::ImageDimensionsTooSmall(nx, ny)); } - let expected_len = usize::try_from(nx) - .ok() - .and_then(|width| width.checked_mul(usize::try_from(ny).ok()?)) - .and_then(|pixels| pixels.checked_mul(RGB_BYTES_PER_PIXEL)) + let expected_len = (u64::from(nx) * u64::from(ny)) + .checked_mul(RGB_BYTES_PER_PIXEL) + .and_then(|bytes| usize::try_from(bytes).ok()) .ok_or(MtmdBitmapError::ImageDimensionsOverflow { nx, ny })?; if data.len() != expected_len { return Err(MtmdBitmapError::InvalidDataSize); } - let bitmap = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_init(nx, ny, data.as_ptr()) }; - - let bitmap = NonNull::new(bitmap).ok_or(MtmdBitmapError::BitmapDecodeFailed)?; + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init( + nx, + ny, + data.as_ptr(), + &raw mut out_bitmap, + &raw mut out_error, + ) + }; - Ok(Self { bitmap }) + bitmap_init_status_to_result(status, out_bitmap, out_error) } /// # Errors /// - /// * `NullResult` - Underlying C function returned null - /// + /// Returns the [`MtmdBitmapError`] variant matching the wrapper's status code. pub fn from_audio_data(data: &[f32]) -> Result { - let bitmap = unsafe { - llama_cpp_bindings_sys::mtmd_bitmap_init_from_audio(data.len(), data.as_ptr()) + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_audio( + data.len(), + data.as_ptr(), + &raw mut out_bitmap, + &raw mut out_error, + ) }; - let bitmap = NonNull::new(bitmap).ok_or(MtmdBitmapError::BitmapDecodeFailed)?; - - Ok(Self { bitmap }) + bitmap_init_from_audio_status_to_result(status, out_bitmap, out_error) } /// # Errors @@ -203,8 +314,8 @@ impl MtmdBitmap { /// Returns an [`MtmdBitmapError`] variant matching the wrapper's status code. pub fn from_file(ctx: &MtmdContext, path: &str) -> Result { let path_cstr = CString::new(path)?; - let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = std::ptr::null_mut(); - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_file( @@ -222,8 +333,8 @@ impl MtmdBitmap { /// /// Returns an [`MtmdBitmapError`] variant matching the wrapper's status code. pub fn from_buffer(ctx: &MtmdContext, data: &[u8]) -> Result { - let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = std::ptr::null_mut(); - let mut out_error: *mut c_char = std::ptr::null_mut(); + let mut out_bitmap: *mut llama_cpp_bindings_sys::mtmd_bitmap = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_mtmd_bitmap_init_from_buf( @@ -248,11 +359,17 @@ impl MtmdBitmap { unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_ny(self.bitmap.as_ptr()) } } + /// The bitmap's bytes, or none for a placeholder bitmap, whose data mtmd reports as null. #[must_use] pub fn data(&self) -> &[u8] { - let ptr = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_data(self.bitmap.as_ptr()) }; + let data = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_data(self.bitmap.as_ptr()) }; let len = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_n_bytes(self.bitmap.as_ptr()) }; - unsafe { slice::from_raw_parts(ptr, len) } + + if data.is_null() { + return &[]; + } + + unsafe { slice::from_raw_parts(data, len) } } #[must_use] @@ -260,18 +377,22 @@ impl MtmdBitmap { unsafe { llama_cpp_bindings_sys::mtmd_bitmap_is_audio(self.bitmap.as_ptr()) } } - #[must_use] - pub fn id(&self) -> Option { - let ptr = unsafe { llama_cpp_bindings_sys::mtmd_bitmap_get_id(self.bitmap.as_ptr()) }; - - cstr_ptr_to_optional_string(ptr) + /// # Errors + /// + /// Returns [`Utf8Error`] when the id is not valid UTF-8. + pub fn id(&self) -> Result, Utf8Error> { + unsafe { + optional_mtmd_id(llama_cpp_bindings_sys::mtmd_bitmap_get_id( + self.bitmap.as_ptr(), + )) + } } /// # Errors /// /// Returns an error if the ID string contains null bytes. /// - pub fn set_id(&self, id: &str) -> Result<(), std::ffi::NulError> { + pub fn set_id(&self, id: &str) -> Result<(), NulError> { let id_cstr = CString::new(id)?; unsafe { llama_cpp_bindings_sys::mtmd_bitmap_set_id(self.bitmap.as_ptr(), id_cstr.as_ptr()); @@ -289,11 +410,13 @@ impl Drop for MtmdBitmap { #[cfg(test)] mod tests { - use super::MtmdBitmap; + use std::ffi::CString; + + use crate::mtmd::mtmd_bitmap::MtmdBitmap; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; #[test] - fn from_image_data_creates_valid_bitmap() { + fn image_data_makes_an_image_bitmap_of_its_dimensions() { let red_pixel: [u8; 3] = [255, 0, 0]; let image_data: Vec = red_pixel.repeat(4); let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); @@ -303,7 +426,7 @@ mod tests { } #[test] - fn invalid_data_size_returns_error() { + fn image_data_shorter_than_its_dimensions_is_an_invalid_data_size() { let too_short = vec![0u8; 5]; assert_eq!( @@ -349,16 +472,16 @@ mod tests { } #[test] - fn set_id_changes_id() { + fn set_id_replaces_the_bitmap_id() { let image_data = vec![0u8; 12]; let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); bitmap.set_id("test_image").unwrap(); - assert_eq!(bitmap.id().as_deref(), Some("test_image")); + assert_eq!(bitmap.id(), Ok(Some("test_image".to_owned()))); } #[test] - fn from_audio_data_creates_valid_bitmap() { + fn audio_samples_make_an_audio_bitmap() { let audio_samples: Vec = (0u8..100) .map(|index| (f32::from(index) * 0.1).sin()) .collect(); @@ -367,6 +490,13 @@ mod tests { assert!(bitmap.is_audio()); } + #[test] + fn a_placeholder_bitmap_without_samples_holds_no_data() { + let bitmap = MtmdBitmap::from_audio_data(&[]).unwrap(); + + assert_eq!(bitmap.data(), &[] as &[u8]); + } + #[test] fn data_returns_expected_bytes_for_image() { let pixel_data: Vec = vec![255, 0, 0, 0, 255, 0, 0, 0, 255, 128, 128, 128]; @@ -377,57 +507,32 @@ mod tests { } #[test] - fn id_returns_some_by_default() { + fn a_new_bitmap_has_an_empty_id() { let image_data = vec![0u8; 12]; let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); - assert!(bitmap.id().is_some()); + assert_eq!(bitmap.id(), Ok(Some(String::new()))); } #[test] - fn id_returns_custom_value_after_set() { - let image_data = vec![0u8; 12]; - let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); - bitmap.set_id("my_image").unwrap(); - - assert_eq!(bitmap.id(), Some("my_image".to_string())); - } - - #[test] - fn set_id_with_null_byte_returns_error() { + fn set_id_reports_a_nul_byte_in_the_id() { let image_data = vec![0u8; 12]; let bitmap = MtmdBitmap::from_image_data(2, 2, &image_data).unwrap(); let result = bitmap.set_id("id\0null"); - assert!(result.is_err()); - } -} - -#[cfg(test)] -mod cstr_ptr_to_optional_string_tests { - use std::ptr; - - use super::cstr_ptr_to_optional_string; - - #[test] - fn cstr_ptr_to_optional_string_returns_none_for_null() { - assert!(cstr_ptr_to_optional_string(ptr::null()).is_none()); - } - - #[test] - fn cstr_ptr_to_optional_string_returns_some_for_valid() { - let cstr = std::ffi::CString::new("hello").unwrap(); - let result = cstr_ptr_to_optional_string(cstr.as_ptr()); - - assert_eq!(result, Some("hello".to_string())); + assert_eq!(result, Err(CString::new("id\0null").unwrap_err())); } } #[cfg(test)] mod from_file_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::path::PathBuf; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::from_file_status_to_result; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; @@ -444,7 +549,7 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - MtmdBitmapError::FfiContract(crate::FfiContractError { + MtmdBitmapError::FfiContract(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "success status contained a null bitmap", }) @@ -497,10 +602,10 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -519,7 +624,7 @@ mod from_file_status_to_result_tests { assert_eq!( result.unwrap_err(), - MtmdBitmapError::FfiContract(crate::FfiContractError { + MtmdBitmapError::FfiContract(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null ctx argument", }) @@ -527,7 +632,7 @@ mod from_file_status_to_result_tests { } #[test] - fn from_file_status_to_result_maps_null_fname_arg() { + fn from_file_status_to_result_reports_a_null_fname_arg_as_a_contract_violation() { let outcome = unsafe { from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_FNAME_ARG, @@ -539,7 +644,7 @@ mod from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null fname argument", } @@ -549,7 +654,7 @@ mod from_file_status_to_result_tests { } #[test] - fn from_file_status_to_result_maps_null_out_bitmap_arg() { + fn from_file_status_to_result_reports_a_null_out_bitmap_arg_as_a_contract_violation() { let outcome = unsafe { from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_NULL_OUT_BITMAP_ARG, @@ -561,7 +666,7 @@ mod from_file_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_file", detail: "was given a null out_bitmap argument", } @@ -571,7 +676,7 @@ mod from_file_status_to_result_tests { } #[test] - fn from_file_status_to_result_maps_llama_cpp_out_of_memory() { + fn from_file_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = unsafe { from_file_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_LLAMA_CPP_OUT_OF_MEMORY, @@ -582,14 +687,54 @@ mod from_file_status_to_result_tests { }; assert_eq!(outcome.err(), Some(MtmdBitmapError::LlamaCppOutOfMemory)); } + + #[test] + fn from_file_status_llama_cpp_threw_cxx_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"image loader threw".as_ptr()) }; + + let result = unsafe { + from_file_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_FILE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + "/missing/image.png", + ) + }; + + assert_eq!( + result.err(), + Some(MtmdBitmapError::Reported { + message: "image loader threw".to_owned(), + }) + ); + } + + #[test] + fn from_file_unknown_status_is_preserved() { + let result = unsafe { + from_file_status_to_result(255, ptr::null_mut(), ptr::null_mut(), "/missing/image.png") + }; + + assert_eq!( + result.err(), + Some(MtmdBitmapError::FfiStatus(FfiStatusError { + operation: "llama_rs_mtmd_bitmap_init_from_file", + code: 255, + })) + ); + } } #[cfg(test)] mod from_buf_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; - use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; use super::from_buf_status_to_result; use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; @@ -628,7 +773,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "success status contained a null bitmap", } @@ -689,10 +834,10 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - crate::FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ) ); @@ -748,7 +893,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_bitmap_init_from_buf", detail, } @@ -768,7 +913,7 @@ mod from_buf_status_to_result_tests { assert_eq!( result.err(), Some( - crate::FfiStatusError { + FfiStatusError { operation: "llama_rs_mtmd_bitmap_init_from_buf", code: i64::from(unrecognized), } @@ -777,3 +922,147 @@ mod from_buf_status_to_result_tests { ); } } + +#[cfg(test)] +mod bitmap_init_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::ptr; + + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::bitmap_init_from_audio_status_to_result; + use super::bitmap_init_status_to_result; + use crate::mtmd::mtmd_bitmap_error::MtmdBitmapError; + + fn contract_violation(operation: &'static str, detail: &'static str) -> MtmdBitmapError { + FfiContractError { operation, detail }.into() + } + + fn missing_message(operation: &'static str) -> MtmdBitmapError { + CppStringError::Missing(FfiContractError { + operation, + detail: "reported a thrown C++ exception without an error message", + }) + .into() + } + + #[test] + fn every_image_failure_status_maps_to_its_error() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_OK, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_DATA_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_NULL_OUT_BITMAP_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + 255, + ]; + let operation = "llama_rs_mtmd_bitmap_init"; + + assert_eq!( + statuses.map(|status| { + bitmap_init_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() + }), + [ + Some(contract_violation( + operation, + "success status contained a null bitmap" + )), + Some(contract_violation( + operation, + "was given a null data argument" + )), + Some(contract_violation( + operation, + "was given a null out_bitmap argument" + )), + Some(MtmdBitmapError::NotEnoughMemory), + Some(MtmdBitmapError::LlamaCppOutOfMemory), + Some(missing_message(operation)), + Some(MtmdBitmapError::FfiStatus(FfiStatusError { + operation, + code: 255, + })), + ] + ); + } + + #[test] + fn every_audio_failure_status_maps_to_its_error() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_OK, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_DATA_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_NULL_OUT_BITMAP_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, + 255, + ]; + let operation = "llama_rs_mtmd_bitmap_init_from_audio"; + + assert_eq!( + statuses.map(|status| { + bitmap_init_from_audio_status_to_result(status, ptr::null_mut(), ptr::null_mut()) + .err() + }), + [ + Some(contract_violation( + operation, + "success status contained a null bitmap" + )), + Some(contract_violation( + operation, + "was given a null data argument" + )), + Some(contract_violation( + operation, + "was given a null out_bitmap argument" + )), + Some(MtmdBitmapError::NotEnoughMemory), + Some(MtmdBitmapError::LlamaCppOutOfMemory), + Some(missing_message(operation)), + Some(MtmdBitmapError::FfiStatus(FfiStatusError { + operation, + code: 255, + })), + ] + ); + } + + #[test] + fn an_exception_thrown_while_initializing_an_image_bitmap_reports_its_message() { + let image_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"image threw".as_ptr()) }; + + assert_eq!( + bitmap_init_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + image_error, + ) + .err(), + Some(MtmdBitmapError::Reported { + message: "image threw".to_owned(), + }) + ); + } + + #[test] + fn an_exception_thrown_while_initializing_an_audio_bitmap_reports_its_message() { + let audio_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"audio threw".as_ptr()) }; + + assert_eq!( + bitmap_init_from_audio_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_BITMAP_INIT_FROM_AUDIO_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + audio_error, + ) + .err(), + Some(MtmdBitmapError::Reported { + message: "audio threw".to_owned(), + }) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs index 606085f11..c0487d2bd 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_bitmap_error.rs @@ -1,11 +1,17 @@ use std::path::PathBuf; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdBitmapError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("Failed to create CString from bitmap-source path: {0}")] CStringError(#[from] std::ffi::NulError), #[error("Bitmap-source path is not valid UTF-8: {0:?}")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_capability.rs b/llama-cpp-bindings/src/mtmd/mtmd_capability.rs new file mode 100644 index 000000000..ac8e1d7a9 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_capability.rs @@ -0,0 +1,15 @@ +use enumflags2::bitflags; + +#[derive(Eq, PartialEq, Debug, Clone, Copy)] +#[bitflags] +#[repr(u8)] +pub enum MtmdCapability { + Chat = 1 << 0, + Vision = 1 << 1, + Audio = 1 << 2, + Video = 1 << 3, + AudioStream = 1 << 4, + TextGeneration = 1 << 5, + AudioGeneration = 1 << 6, + AudioGenerationSpeakerReference = 1 << 7, +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context.rs b/llama-cpp-bindings/src/mtmd/mtmd_context.rs index 5c85500ff..cfaf54bef 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context.rs @@ -4,11 +4,14 @@ use std::path::PathBuf; use std::ptr; use std::ptr::NonNull; -use llama_cpp_ffi_status::read_and_free_cpp_string; +use enumflags2::BitFlags; -use crate::model::LlamaModel; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::mtmd_bitmap::MtmdBitmap; +use super::mtmd_capability::MtmdCapability; use super::mtmd_context_params::MtmdContextParams; use super::mtmd_encode_error::MtmdEncodeError; use super::mtmd_ffi_input_part::MtmdFfiInputPart; @@ -18,6 +21,38 @@ use super::mtmd_input_chunks::MtmdInputChunks; use super::mtmd_input_part::MtmdInputPart; use super::mtmd_input_text::MtmdInputText; use super::mtmd_tokenize_error::MtmdTokenizeError; +use crate::model::LlamaModel; + +fn capability_flags( + llama_cpp_bindings_sys::mtmd_helper_cap { + chat, + vision, + audio, + video, + audio_stream, + text_gen, + audio_gen, + audio_gen_spk_ref, + }: llama_cpp_bindings_sys::mtmd_helper_cap, +) -> BitFlags { + [ + (chat, MtmdCapability::Chat), + (vision, MtmdCapability::Vision), + (audio, MtmdCapability::Audio), + (video, MtmdCapability::Video), + (audio_stream, MtmdCapability::AudioStream), + (text_gen, MtmdCapability::TextGeneration), + (audio_gen, MtmdCapability::AudioGeneration), + ( + audio_gen_spk_ref, + MtmdCapability::AudioGenerationSpeakerReference, + ), + ] + .into_iter() + .filter(|(is_capable, _capability)| *is_capable) + .map(|(_is_capable, capability)| capability) + .collect() +} fn map_tokenize_status( status: llama_cpp_bindings_sys::llama_rs_mtmd_tokenize_status, @@ -48,28 +83,28 @@ fn map_tokenize_status( Err(MtmdTokenizeError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_BITMAPS_ARG_WHEN_NUM_BITMAPS_NONZERO => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "nonzero bitmap count was observed with a null bitmap array", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_CTX_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_CTX_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null ctx argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_OUTPUT_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_OUTPUT_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null output argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_TEXT_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_TEXT_ARG => Err(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null text argument", } .into()), - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_tokenize", code: i64::from(other), } @@ -85,7 +120,7 @@ fn map_tokenize_from_parts_status( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_OK => Ok(()), llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_REPORTED_MALFORMED_PART => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "llama.cpp reported a malformed part list: a part holding both or neither of text and bitmap, a text part with a null text pointer, or bitmaps that do not line up with their parts", } @@ -116,34 +151,34 @@ fn map_tokenize_from_parts_status( Err(MtmdTokenizeError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_CTX_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null ctx argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_OUTPUT_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null output argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PARTS_ARG_WHEN_NUM_PARTS_NONZERO => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "nonzero part count was observed with a null part array", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PART_ELEMENT => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null element in the parts argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_tokenize_from_parts", code: i64::from(other), } @@ -179,21 +214,19 @@ fn map_encode_chunk_status( }?; Err(MtmdEncodeError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_encode_chunk", - detail: "was given a null ctx argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_mtmd_encode_chunk", + detail: "was given a null ctx argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CHUNK_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null chunk argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_encode_chunk", code: i64::from(other), } @@ -210,7 +243,7 @@ fn map_init_from_file_status( match status { llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_OK => { let context = NonNull::new(out_ctx).ok_or_else(|| { - MtmdInitError::from(crate::FfiContractError { + MtmdInitError::from(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "success status contained a null multimodal context", }) @@ -239,27 +272,27 @@ fn map_init_from_file_status( Err(MtmdInitError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_MMPROJ_PATH_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null mmproj_path argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_TEXT_MODEL_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null text_model argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_OUT_CTX_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null out_ctx argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_mtmd_init_from_file", code: i64::from(other), } @@ -285,7 +318,7 @@ impl MtmdContext { params: &MtmdContextParams, ) -> Result { let path_cstr = CString::new(mmproj_path)?; - let ctx_params = llama_cpp_bindings_sys::mtmd_context_params::from(params); + let ctx_params = llama_cpp_bindings_sys::mtmd_context_params::try_from(params)?; let mut out_ctx: *mut llama_cpp_bindings_sys::mtmd_context = ptr::null_mut(); let mut out_error: *mut c_char = ptr::null_mut(); @@ -319,13 +352,13 @@ impl MtmdContext { } #[must_use] - pub fn support_vision(&self) -> bool { - unsafe { llama_cpp_bindings_sys::mtmd_support_vision(self.context.as_ptr()) } - } - - #[must_use] - pub fn support_audio(&self) -> bool { - unsafe { llama_cpp_bindings_sys::mtmd_support_audio(self.context.as_ptr()) } + pub fn capabilities(&self, model: &LlamaModel) -> BitFlags { + capability_flags(unsafe { + llama_cpp_bindings_sys::mtmd_helper_get_capabilities( + model.model.as_ptr(), + self.context.as_ptr(), + ) + }) } #[must_use] @@ -443,10 +476,13 @@ impl Drop for MtmdContext { #[cfg(test)] mod map_tokenize_from_parts_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; - use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; use super::map_tokenize_from_parts_status; use crate::mtmd::mtmd_tokenize_error::MtmdTokenizeError; @@ -464,7 +500,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn tokenize_from_parts_status_maps_preprocessing_or_tokenization_failure() { + fn tokenize_from_parts_status_reports_a_media_preprocessing_or_tokenization_failure() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_REPORTED_PREPROCESSING_OR_TOKENIZATION_FAILURE, @@ -500,7 +536,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn tokenize_from_parts_status_maps_llama_cpp_out_of_memory() { + fn tokenize_from_parts_status_reports_llama_cpp_running_out_of_memory() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_OUT_OF_MEMORY, @@ -535,7 +571,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_llama_cpp_reported_malformed_part() { + fn map_tokenize_from_parts_status_reports_a_malformed_part_list_as_a_contract_violation() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_REPORTED_MALFORMED_PART, @@ -544,7 +580,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "llama.cpp reported a malformed part list: a part holding both or neither of text and bitmap, a text part with a null text pointer, or bitmaps that do not line up with their parts", } @@ -554,7 +590,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_null_part_element() { + fn map_tokenize_from_parts_status_reports_a_null_part_element_as_a_contract_violation() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PART_ELEMENT, @@ -563,7 +599,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null element in the parts argument", } @@ -573,7 +609,8 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_llama_cpp_threw_cxx_exception_without_a_message() { + fn map_tokenize_from_parts_status_reports_an_exception_without_a_message_as_a_contract_violation() + { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_LLAMA_CPP_THREW_CXX_EXCEPTION, @@ -582,17 +619,17 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ) ); } #[test] - fn map_tokenize_from_parts_status_maps_null_ctx_arg() { + fn map_tokenize_from_parts_status_reports_a_null_ctx_arg_as_a_contract_violation() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_CTX_ARG, @@ -601,7 +638,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null ctx argument", } @@ -611,7 +648,7 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_null_output_arg() { + fn map_tokenize_from_parts_status_reports_a_null_output_arg_as_a_contract_violation() { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_OUTPUT_ARG, @@ -620,7 +657,7 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "was given a null output argument", } @@ -630,7 +667,8 @@ mod map_tokenize_from_parts_status_tests { } #[test] - fn map_tokenize_from_parts_status_maps_null_parts_arg_when_num_parts_nonzero() { + fn map_tokenize_from_parts_status_reports_a_null_parts_arg_with_a_nonzero_part_count_as_a_contract_violation() + { assert_eq!( map_tokenize_from_parts_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_FROM_PARTS_NULL_PARTS_ARG_WHEN_NUM_PARTS_NONZERO, @@ -639,16 +677,20 @@ mod map_tokenize_from_parts_status_tests { ) .err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize_from_parts", detail: "nonzero part count was observed with a null part array", } .into() ) ); + } + + #[test] + fn map_tokenize_from_parts_status_preserves_an_unknown_status() { assert_eq!( map_tokenize_from_parts_status(255, 0, ptr::null_mut()).err(), - Some(MtmdTokenizeError::FfiStatus(crate::FfiStatusError { + Some(MtmdTokenizeError::FfiStatus(FfiStatusError { operation: "llama_rs_mtmd_tokenize_from_parts", code: 255, })) @@ -658,13 +700,17 @@ mod map_tokenize_from_parts_status_tests { #[cfg(test)] mod map_tokenize_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::map_tokenize_status; use crate::mtmd::mtmd_tokenize_error::MtmdTokenizeError; #[test] - fn tokenize_status_maps_bitmap_count_mismatch() { + fn tokenize_status_reports_a_bitmap_count_that_does_not_match_the_marker_count() { let result = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_REPORTED_BITMAP_COUNT_DOES_NOT_MATCH_MARKER_COUNT, 0, @@ -678,7 +724,7 @@ mod map_tokenize_status_tests { } #[test] - fn tokenize_status_maps_preprocessing_or_tokenization_failure() { + fn tokenize_status_reports_a_media_preprocessing_or_tokenization_failure() { let result = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_REPORTED_PREPROCESSING_OR_TOKENIZATION_FAILURE, 0, @@ -692,7 +738,7 @@ mod map_tokenize_status_tests { } #[test] - fn tokenize_status_maps_unknown_status_with_value() { + fn tokenize_status_maps_an_undocumented_code_with_its_value() { let result = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_RETURNED_UNDOCUMENTED_NONZERO_CODE, 42, @@ -725,7 +771,7 @@ mod map_tokenize_status_tests { } #[test] - fn tokenize_status_maps_cxx_exception_to_without_a_message_is_a_contract_error() { + fn tokenize_status_reports_an_exception_without_a_message_as_a_contract_violation() { let result = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, 0, @@ -734,10 +780,10 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -752,7 +798,7 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(MtmdTokenizeError::FfiContract(crate::FfiContractError { + Err(MtmdTokenizeError::FfiContract(FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "nonzero bitmap count was observed with a null bitmap array", })) @@ -765,7 +811,7 @@ mod map_tokenize_status_tests { assert_eq!( result, - Err(MtmdTokenizeError::FfiStatus(crate::FfiStatusError { + Err(MtmdTokenizeError::FfiStatus(FfiStatusError { operation: "llama_rs_mtmd_tokenize", code: 255, })) @@ -773,7 +819,7 @@ mod map_tokenize_status_tests { } #[test] - fn map_tokenize_status_maps_null_ctx_arg() { + fn map_tokenize_status_reports_a_null_ctx_arg_as_a_contract_violation() { let outcome = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_CTX_ARG, 0, @@ -782,7 +828,7 @@ mod map_tokenize_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null ctx argument", } @@ -792,7 +838,7 @@ mod map_tokenize_status_tests { } #[test] - fn map_tokenize_status_maps_null_output_arg() { + fn map_tokenize_status_reports_a_null_output_arg_as_a_contract_violation() { let outcome = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_OUTPUT_ARG, 0, @@ -801,7 +847,7 @@ mod map_tokenize_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null output argument", } @@ -811,7 +857,7 @@ mod map_tokenize_status_tests { } #[test] - fn map_tokenize_status_maps_null_text_arg() { + fn map_tokenize_status_reports_a_null_text_arg_as_a_contract_violation() { let outcome = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_NULL_TEXT_ARG, 0, @@ -820,7 +866,7 @@ mod map_tokenize_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_tokenize", detail: "was given a null text argument", } @@ -830,7 +876,7 @@ mod map_tokenize_status_tests { } #[test] - fn map_tokenize_status_maps_llama_cpp_out_of_memory() { + fn map_tokenize_status_reports_llama_cpp_running_out_of_memory() { let outcome = map_tokenize_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_OUT_OF_MEMORY, 0, @@ -838,12 +884,33 @@ mod map_tokenize_status_tests { ); assert_eq!(outcome.err(), Some(MtmdTokenizeError::LlamaCppOutOfMemory)); } + + #[test] + fn map_tokenize_status_maps_a_thrown_exception_with_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"tokenizer threw".as_ptr()) }; + + assert_eq!( + map_tokenize_status( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_TOKENIZE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ), + Err(MtmdTokenizeError::Reported { + message: "tokenizer threw".to_owned(), + }) + ); + } } #[cfg(test)] mod map_encode_chunk_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::map_encode_chunk_status; use crate::mtmd::mtmd_encode_error::MtmdEncodeError; @@ -881,7 +948,7 @@ mod map_encode_chunk_status_tests { } #[test] - fn encode_chunk_status_maps_cxx_exception_to_without_a_message_is_a_contract_error() { + fn encode_chunk_status_reports_an_exception_without_a_message_as_a_contract_violation() { let result = map_encode_chunk_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_LLAMA_CPP_THREW_CXX_EXCEPTION, 0, @@ -890,10 +957,10 @@ mod map_encode_chunk_status_tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } @@ -904,7 +971,7 @@ mod map_encode_chunk_status_tests { assert_eq!( result, - Err(MtmdEncodeError::FfiStatus(crate::FfiStatusError { + Err(MtmdEncodeError::FfiStatus(FfiStatusError { operation: "llama_rs_mtmd_encode_chunk", code: 255, })) @@ -912,7 +979,7 @@ mod map_encode_chunk_status_tests { } #[test] - fn map_encode_chunk_status_maps_null_ctx_arg() { + fn map_encode_chunk_status_reports_a_null_ctx_arg_as_a_contract_violation() { let outcome = map_encode_chunk_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CTX_ARG, 0, @@ -921,7 +988,7 @@ mod map_encode_chunk_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null ctx argument", } @@ -931,7 +998,7 @@ mod map_encode_chunk_status_tests { } #[test] - fn map_encode_chunk_status_maps_null_chunk_arg() { + fn map_encode_chunk_status_reports_a_null_chunk_arg_as_a_contract_violation() { let outcome = map_encode_chunk_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_NULL_CHUNK_ARG, 0, @@ -940,7 +1007,7 @@ mod map_encode_chunk_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_encode_chunk", detail: "was given a null chunk argument", } @@ -950,7 +1017,7 @@ mod map_encode_chunk_status_tests { } #[test] - fn map_encode_chunk_status_maps_llama_cpp_out_of_memory() { + fn map_encode_chunk_status_reports_llama_cpp_running_out_of_memory() { let outcome = map_encode_chunk_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_LLAMA_CPP_OUT_OF_MEMORY, 0, @@ -958,12 +1025,33 @@ mod map_encode_chunk_status_tests { ); assert_eq!(outcome.err(), Some(MtmdEncodeError::LlamaCppOutOfMemory)); } + + #[test] + fn map_encode_chunk_status_maps_a_thrown_exception_with_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"encoder threw".as_ptr()) }; + + assert_eq!( + map_encode_chunk_status( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_ENCODE_CHUNK_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + out_error, + ), + Err(MtmdEncodeError::Reported { + message: "encoder threw".to_owned(), + }) + ); + } } #[cfg(test)] mod map_init_from_file_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::map_init_from_file_status; use crate::mtmd::mtmd_init_error::MtmdInitError; @@ -978,7 +1066,7 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - MtmdInitError::FfiContract(crate::FfiContractError { + MtmdInitError::FfiContract(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "success status contained a null multimodal context", }) @@ -998,7 +1086,7 @@ mod map_init_from_file_status_tests { } #[test] - fn init_from_file_status_maps_cxx_exception_to_without_a_message_is_a_contract_error() { + fn init_from_file_status_reports_an_exception_without_a_message_as_a_contract_violation() { let result = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), @@ -1008,10 +1096,10 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -1023,7 +1111,7 @@ mod map_init_from_file_status_tests { assert_eq!( result.unwrap_err(), - MtmdInitError::FfiStatus(crate::FfiStatusError { + MtmdInitError::FfiStatus(FfiStatusError { operation: "llama_rs_mtmd_init_from_file", code: 255, }) @@ -1031,7 +1119,7 @@ mod map_init_from_file_status_tests { } #[test] - fn map_init_from_file_status_maps_null_mmproj_path_arg() { + fn map_init_from_file_status_reports_a_null_mmproj_path_arg_as_a_contract_violation() { let outcome = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_MMPROJ_PATH_ARG, ptr::null_mut(), @@ -1041,7 +1129,7 @@ mod map_init_from_file_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null mmproj_path argument", } @@ -1051,7 +1139,7 @@ mod map_init_from_file_status_tests { } #[test] - fn map_init_from_file_status_maps_null_text_model_arg() { + fn map_init_from_file_status_reports_a_null_text_model_arg_as_a_contract_violation() { let outcome = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_TEXT_MODEL_ARG, ptr::null_mut(), @@ -1061,7 +1149,7 @@ mod map_init_from_file_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null text_model argument", } @@ -1071,7 +1159,7 @@ mod map_init_from_file_status_tests { } #[test] - fn map_init_from_file_status_maps_null_out_ctx_arg() { + fn map_init_from_file_status_reports_a_null_out_ctx_arg_as_a_contract_violation() { let outcome = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_NULL_OUT_CTX_ARG, ptr::null_mut(), @@ -1081,7 +1169,7 @@ mod map_init_from_file_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_mtmd_init_from_file", detail: "was given a null out_ctx argument", } @@ -1091,7 +1179,7 @@ mod map_init_from_file_status_tests { } #[test] - fn map_init_from_file_status_maps_llama_cpp_out_of_memory() { + fn map_init_from_file_status_reports_llama_cpp_running_out_of_memory() { let outcome = map_init_from_file_status( llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), @@ -1100,4 +1188,114 @@ mod map_init_from_file_status_tests { ); assert_eq!(outcome.err(), Some(MtmdInitError::LlamaCppOutOfMemory)); } + + #[test] + fn map_init_from_file_status_maps_a_thrown_exception_with_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"projector threw".as_ptr()) }; + + assert_eq!( + map_init_from_file_status( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INIT_FROM_FILE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + "mmproj.gguf", + ) + .err(), + Some(MtmdInitError::Reported { + message: "projector threw".to_owned(), + }) + ); + } +} + +#[cfg(test)] +mod capability_flags_tests { + use enumflags2::BitFlags; + + use super::capability_flags; + use crate::mtmd::mtmd_capability::MtmdCapability; + + const NO_CAPABILITIES: llama_cpp_bindings_sys::mtmd_helper_cap = + llama_cpp_bindings_sys::mtmd_helper_cap { + chat: false, + vision: false, + audio: false, + video: false, + audio_stream: false, + text_gen: false, + audio_gen: false, + audio_gen_spk_ref: false, + }; + + #[test] + fn a_model_without_capabilities_has_no_flags() { + assert_eq!(capability_flags(NO_CAPABILITIES), BitFlags::empty()); + } + + #[test] + fn every_capability_of_llama_cpp_maps_to_its_own_flag() { + let single_capabilities = [ + llama_cpp_bindings_sys::mtmd_helper_cap { + chat: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + vision: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + audio: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + video: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + audio_stream: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + text_gen: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + audio_gen: true, + ..NO_CAPABILITIES + }, + llama_cpp_bindings_sys::mtmd_helper_cap { + audio_gen_spk_ref: true, + ..NO_CAPABILITIES + }, + ]; + + assert_eq!( + single_capabilities.map(capability_flags), + [ + BitFlags::from_flag(MtmdCapability::Chat), + BitFlags::from_flag(MtmdCapability::Vision), + BitFlags::from_flag(MtmdCapability::Audio), + BitFlags::from_flag(MtmdCapability::Video), + BitFlags::from_flag(MtmdCapability::AudioStream), + BitFlags::from_flag(MtmdCapability::TextGeneration), + BitFlags::from_flag(MtmdCapability::AudioGeneration), + BitFlags::from_flag(MtmdCapability::AudioGenerationSpeakerReference), + ] + ); + } + + #[test] + fn a_text_to_speech_model_generates_audio_but_cannot_chat() { + let text_to_speech = llama_cpp_bindings_sys::mtmd_helper_cap { + audio_gen: true, + audio_gen_spk_ref: true, + ..NO_CAPABILITIES + }; + + assert_eq!( + capability_flags(text_to_speech), + MtmdCapability::AudioGeneration | MtmdCapability::AudioGenerationSpeakerReference + ); + } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs b/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs index b850580bd..d7898fea9 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_context_params.rs @@ -1,11 +1,16 @@ use std::ffi::{CStr, CString}; -#[derive(Debug, Clone)] +use super::mtmd_context_params_error::MtmdContextParamsError; +use super::mtmd_image_token_limit::MtmdImageTokenLimit; + +#[derive(Debug, Clone, PartialEq, Eq)] pub struct MtmdContextParams { pub use_gpu: bool, pub print_timings: bool, pub n_threads: i32, pub media_marker: CString, + pub image_min_tokens: MtmdImageTokenLimit, + pub image_max_tokens: MtmdImageTokenLimit, } impl Default for MtmdContextParams { @@ -14,22 +19,38 @@ impl Default for MtmdContextParams { } } -impl From<&MtmdContextParams> for llama_cpp_bindings_sys::mtmd_context_params { - fn from(params: &MtmdContextParams) -> Self { +impl TryFrom<&MtmdContextParams> for llama_cpp_bindings_sys::mtmd_context_params { + type Error = MtmdContextParamsError; + + fn try_from(params: &MtmdContextParams) -> Result { let mut context = unsafe { llama_cpp_bindings_sys::mtmd_context_params_default() }; let MtmdContextParams { use_gpu, print_timings, n_threads, media_marker, + image_min_tokens, + image_max_tokens, } = params; context.use_gpu = *use_gpu; context.print_timings = *print_timings; context.n_threads = *n_threads; context.media_marker = media_marker.as_ptr(); + context.image_min_tokens = image_min_tokens.to_c_int().map_err(|source| { + MtmdContextParamsError::ImageMinTokensExceedCInt { + limit: *image_min_tokens, + source, + } + })?; + context.image_max_tokens = image_max_tokens.to_c_int().map_err(|source| { + MtmdContextParamsError::ImageMaxTokensExceedCInt { + limit: *image_max_tokens, + source, + } + })?; - context + Ok(context) } } @@ -40,6 +61,72 @@ impl From for MtmdContextParams { print_timings: params.print_timings, n_threads: params.n_threads, media_marker: unsafe { CStr::from_ptr(params.media_marker) }.to_owned(), + image_min_tokens: MtmdImageTokenLimit::from_c_int(params.image_min_tokens), + image_max_tokens: MtmdImageTokenLimit::from_c_int(params.image_max_tokens), } } } + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use crate::mtmd::mtmd_context_params::MtmdContextParams; + use crate::mtmd::mtmd_context_params_error::MtmdContextParamsError; + use crate::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; + + const SIXTEEN: NonZeroU32 = NonZeroU32::new(16).unwrap(); + const SIXTY_FOUR: NonZeroU32 = NonZeroU32::new(64).unwrap(); + const UNREPRESENTABLE_LIMIT: MtmdImageTokenLimit = MtmdImageTokenLimit::Tokens(NonZeroU32::MAX); + + #[test] + fn image_token_limits_reach_the_c_parameters() { + let params = MtmdContextParams { + image_min_tokens: MtmdImageTokenLimit::Tokens(SIXTEEN), + image_max_tokens: MtmdImageTokenLimit::Tokens(SIXTY_FOUR), + ..MtmdContextParams::default() + }; + + assert_eq!( + llama_cpp_bindings_sys::mtmd_context_params::try_from(¶ms) + .map(|c_params| [c_params.image_min_tokens, c_params.image_max_tokens]), + Ok([16, 64]) + ); + } + + #[test] + fn an_unrepresentable_image_min_token_limit_is_refused() { + let params = MtmdContextParams { + image_min_tokens: UNREPRESENTABLE_LIMIT, + ..MtmdContextParams::default() + }; + + assert_eq!( + llama_cpp_bindings_sys::mtmd_context_params::try_from(¶ms).err(), + i32::try_from(u32::MAX).err().map(|source| { + MtmdContextParamsError::ImageMinTokensExceedCInt { + limit: UNREPRESENTABLE_LIMIT, + source, + } + }) + ); + } + + #[test] + fn an_unrepresentable_image_max_token_limit_is_refused() { + let params = MtmdContextParams { + image_max_tokens: UNREPRESENTABLE_LIMIT, + ..MtmdContextParams::default() + }; + + assert_eq!( + llama_cpp_bindings_sys::mtmd_context_params::try_from(¶ms).err(), + i32::try_from(u32::MAX).err().map(|source| { + MtmdContextParamsError::ImageMaxTokensExceedCInt { + limit: UNREPRESENTABLE_LIMIT, + source, + } + }) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_context_params_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_context_params_error.rs new file mode 100644 index 000000000..d4ad93886 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_context_params_error.rs @@ -0,0 +1,19 @@ +use std::num::TryFromIntError; + +use super::mtmd_image_token_limit::MtmdImageTokenLimit; + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum MtmdContextParamsError { + #[error("the image minimum token limit {limit:?} exceeds what a C int holds")] + ImageMinTokensExceedCInt { + limit: MtmdImageTokenLimit, + #[source] + source: TryFromIntError, + }, + #[error("the image maximum token limit {limit:?} exceeds what a C int holds")] + ImageMaxTokensExceedCInt { + limit: MtmdImageTokenLimit, + #[source] + source: TryFromIntError, + }, +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs b/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs index facff884a..2ae3bb94b 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_default_marker.rs @@ -1,5 +1,5 @@ use std::ffi::CStr; -use std::os::raw::c_char; +use std::ffi::c_char; use crate::mtmd::mtmd_default_marker_error::MtmdDefaultMarkerError; @@ -21,16 +21,16 @@ pub fn mtmd_default_marker() -> Result<&'static str, MtmdDefaultMarkerError> { #[cfg(test)] mod tests { - use std::os::raw::c_char; + use std::ffi::c_char; + use std::str; use super::marker_bytes_to_str; - use super::mtmd_default_marker; + use crate::mtmd::mtmd_default_marker::mtmd_default_marker; use crate::mtmd::mtmd_default_marker_error::MtmdDefaultMarkerError; #[test] - fn returns_non_empty_marker() { - let marker = mtmd_default_marker().expect("llama.cpp marker must be valid UTF-8"); - assert_ne!(marker, ""); + fn the_default_marker_is_the_media_marker() { + assert_eq!(mtmd_default_marker(), Ok("<__media__>")); } #[test] @@ -38,6 +38,11 @@ mod tests { let invalid: [u8; 3] = [0xFF, 0xFE, 0x00]; let result = unsafe { marker_bytes_to_str(invalid.as_ptr().cast::()) }; - assert!(matches!(result, Err(MtmdDefaultMarkerError::NotUtf8(_)))); + assert_eq!( + result.err(), + str::from_utf8(&invalid[..2]) + .err() + .map(MtmdDefaultMarkerError::NotUtf8) + ); } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs index 0906829ab..e76405dc8 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_encode_error.rs @@ -1,9 +1,15 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdEncodeError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), #[error("multimodal chunk encoding failed with code: {code}")] EncodingFailed { code: i32 }, #[error("not enough memory")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs index 5f5663068..dab1eb4bd 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs @@ -1,16 +1,25 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; use crate::mtmd::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdEvalError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("batch size {requested} exceeds context batch size {context_max}")] BatchSizeExceedsContextLimit { requested: i32, context_max: u32 }, #[error("batch size {requested} must be positive")] NonPositiveBatchSize { requested: i32 }, + #[error(transparent)] + SequenceIdOutOfRange(#[from] SequenceIdOutOfRange), #[error( "a chunk decoded non-causally has {} tokens but a single decode fits {}", .0.chunk_tokens, diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs new file mode 100644 index 000000000..1fac6b1d8 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_limits.rs @@ -0,0 +1,114 @@ +use std::num::NonZeroU32; + +use super::mtmd_eval_error::MtmdEvalError; +use crate::context::LlamaContext; +use crate::context::context_limits::ContextLimits; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MtmdEvalLimits { + pub context: ContextLimits, +} + +impl MtmdEvalLimits { + #[must_use] + pub const fn of(llama_ctx: &LlamaContext) -> Self { + Self { + context: ContextLimits::of(llama_ctx), + } + } + + /// Returns `n_batch` as a token count once both it and `seq_id` fit the context, so the mtmd + /// helper never adds a token its batch refuses. + /// + /// # Errors + /// + /// Returns [`MtmdEvalError::NonPositiveBatchSize`] when `n_batch` is not positive, + /// [`MtmdEvalError::BatchSizeExceedsContextLimit`] when it exceeds the context's batch size, + /// or [`MtmdEvalError::SequenceIdOutOfRange`] when `seq_id` names no sequence of the context. + pub fn batch_tokens( + &self, + n_batch: i32, + seq_id: llama_cpp_bindings_sys::llama_seq_id, + ) -> Result { + let batch_tokens = u32::try_from(n_batch) + .ok() + .and_then(NonZeroU32::new) + .ok_or(MtmdEvalError::NonPositiveBatchSize { requested: n_batch })?; + + if batch_tokens.get() > self.context.n_batch { + return Err(MtmdEvalError::BatchSizeExceedsContextLimit { + requested: n_batch, + context_max: self.context.n_batch, + }); + } + + self.context.ensure_sequence_id(seq_id)?; + + Ok(batch_tokens) + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use crate::context::context_limits::ContextLimits; + use crate::error::sequence_id_out_of_range::SequenceIdOutOfRange; + use crate::mtmd::mtmd_eval_error::MtmdEvalError; + use crate::mtmd::mtmd_eval_limits::MtmdEvalLimits; + + const LIMITS: MtmdEvalLimits = MtmdEvalLimits { + context: ContextLimits { + n_batch: 512, + n_ubatch: 128, + n_seq_max: 2, + n_vocab: 100, + }, + }; + + #[test] + fn a_batch_within_the_context_for_an_existing_sequence_is_accepted() { + assert_eq!( + LIMITS.batch_tokens(512, 1), + Ok(NonZeroU32::new(512).unwrap()) + ); + } + + #[test] + fn a_zero_batch_is_rejected() { + assert_eq!( + LIMITS.batch_tokens(0, 0), + Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) + ); + } + + #[test] + fn a_negative_batch_is_rejected() { + assert_eq!( + LIMITS.batch_tokens(-1, 0), + Err(MtmdEvalError::NonPositiveBatchSize { requested: -1 }) + ); + } + + #[test] + fn a_batch_larger_than_the_context_batch_is_rejected() { + assert_eq!( + LIMITS.batch_tokens(513, 0), + Err(MtmdEvalError::BatchSizeExceedsContextLimit { + requested: 513, + context_max: 512, + }) + ); + } + + #[test] + fn a_sequence_id_the_context_lacks_is_rejected() { + assert_eq!( + LIMITS.batch_tokens(64, 2), + Err(MtmdEvalError::SequenceIdOutOfRange(SequenceIdOutOfRange { + seq_id: 2, + n_seq_max: 2, + })) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs b/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs new file mode 100644 index 000000000..9f945c36d --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_image_token_limit.rs @@ -0,0 +1,98 @@ +use std::ffi::c_int; +use std::num::NonZeroU32; +use std::num::TryFromIntError; + +/// A bound on the number of tokens the vision encoder produces for one image of a model with +/// dynamic resolution. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MtmdImageTokenLimit { + ModelMetadata, + Tokens(NonZeroU32), +} + +impl MtmdImageTokenLimit { + #[must_use] + pub fn from_c_int(raw: c_int) -> Self { + u32::try_from(raw) + .ok() + .and_then(NonZeroU32::new) + .map_or(Self::ModelMetadata, Self::Tokens) + } + + /// # Errors + /// + /// Returns the conversion error when the token count exceeds what a C `int` holds. + pub fn to_c_int(self) -> Result { + match self { + Self::ModelMetadata => Ok(-1), + Self::Tokens(tokens) => c_int::try_from(tokens.get()), + } + } + + #[must_use] + pub fn at_most(self, ceiling: NonZeroU32) -> Self { + match self { + Self::ModelMetadata => Self::ModelMetadata, + Self::Tokens(tokens) => Self::Tokens(tokens.min(ceiling)), + } + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use crate::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; + + const SIXTY_FOUR: NonZeroU32 = NonZeroU32::new(64).unwrap(); + + #[test] + fn a_non_positive_c_value_reads_the_limit_from_the_model_metadata() { + assert_eq!( + [-1, 0].map(MtmdImageTokenLimit::from_c_int), + [MtmdImageTokenLimit::ModelMetadata; 2] + ); + } + + #[test] + fn a_positive_c_value_is_a_token_limit() { + assert_eq!( + MtmdImageTokenLimit::from_c_int(64), + MtmdImageTokenLimit::Tokens(SIXTY_FOUR) + ); + } + + #[test] + fn the_model_metadata_limit_is_the_c_value_minus_one() { + assert_eq!(MtmdImageTokenLimit::ModelMetadata.to_c_int(), Ok(-1)); + } + + #[test] + fn a_token_limit_is_its_c_value() { + assert_eq!(MtmdImageTokenLimit::Tokens(SIXTY_FOUR).to_c_int(), Ok(64)); + } + + #[test] + fn a_token_limit_beyond_a_c_int_does_not_convert() { + assert_eq!( + MtmdImageTokenLimit::Tokens(NonZeroU32::MAX).to_c_int(), + i32::try_from(u32::MAX) + ); + } + + #[test] + fn a_token_limit_above_the_ceiling_drops_to_the_ceiling() { + assert_eq!( + MtmdImageTokenLimit::Tokens(NonZeroU32::MAX).at_most(SIXTY_FOUR), + MtmdImageTokenLimit::Tokens(SIXTY_FOUR) + ); + } + + #[test] + fn the_model_metadata_limit_stays_under_a_ceiling() { + assert_eq!( + MtmdImageTokenLimit::ModelMetadata.at_most(SIXTY_FOUR), + MtmdImageTokenLimit::ModelMetadata + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs index 6f9f45cf1..9fac767e0 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_init_error.rs @@ -1,11 +1,21 @@ use std::path::PathBuf; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use super::mtmd_context_params_error::MtmdContextParamsError; + #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInitError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + ContextParams(#[from] MtmdContextParamsError), #[error("Failed to create CString from mmproj path: {0}")] CStringError(#[from] std::ffi::NulError), #[error("Mmproj path is not valid UTF-8: {0:?}")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index 4c38746bc..b09ac0f86 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -1,8 +1,12 @@ -use std::ffi::CStr; +use std::ffi::c_char; +use std::ptr; use std::ptr::NonNull; use std::slice; +use std::str::Utf8Error; -use crate::token::LlamaToken; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; @@ -10,6 +14,8 @@ use super::mtmd_input_chunk_error::MtmdInputChunkError; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; use super::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; +use super::optional_mtmd_id::optional_mtmd_id; +use crate::token::LlamaToken; /// # Safety /// @@ -50,6 +56,62 @@ fn non_causal_chunk_micro_batch_error( None } +fn input_chunk_copy_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_input_chunk_copy_status, + out_chunk: *mut llama_cpp_bindings_sys::mtmd_input_chunk, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_mtmd_input_chunk_copy"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK => NonNull::new(out_chunk) + .map(|chunk| MtmdInputChunk { chunk, owned: true }) + .ok_or_else(|| { + FfiContractError { + operation: OPERATION, + detail: "success status contained a null chunk", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG => { + Err(FfiContractError { + operation: OPERATION, + detail: "was given a null chunk argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG => { + Err(FfiContractError { + operation: OPERATION, + detail: "was given a null out_chunk argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdInputChunkError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdInputChunkError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdInputChunkError::Reported { message }) + } + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + #[derive(Debug)] pub struct MtmdInputChunk { pub chunk: NonNull, @@ -60,8 +122,9 @@ impl MtmdInputChunk { /// # Errors /// Returns an error if the chunk type is unknown. pub fn chunk_type(&self) -> Result { - let chunk_type = - unsafe { llama_cpp_bindings_sys::mtmd_input_chunk_get_type(self.chunk.as_ptr()) }; + let chunk_type = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_input_chunk_get_type(self.chunk.as_ptr()) + }; MtmdInputChunkType::try_from(chunk_type) } @@ -92,27 +155,32 @@ impl MtmdInputChunk { unsafe { llama_cpp_bindings_sys::mtmd_input_chunk_get_n_pos(self.chunk.as_ptr()) } } - #[must_use] - pub fn id(&self) -> Option { - let ptr = unsafe { llama_cpp_bindings_sys::mtmd_input_chunk_get_id(self.chunk.as_ptr()) }; - if ptr.is_null() { - None - } else { - unsafe { CStr::from_ptr(ptr) } - .to_string_lossy() - .into_owned() - .into() + /// # Errors + /// + /// Returns [`Utf8Error`] when the id is not valid UTF-8. + pub fn id(&self) -> Result, Utf8Error> { + unsafe { + optional_mtmd_id(llama_cpp_bindings_sys::mtmd_input_chunk_get_id( + self.chunk.as_ptr(), + )) } } /// # Errors /// - /// Returns `MtmdInputChunkError::ChunkOperationFailed` if copying fails. + /// Returns the [`MtmdInputChunkError`] variant matching the wrapper's status code. pub fn copy(&self) -> Result { - let chunk = unsafe { llama_cpp_bindings_sys::mtmd_input_chunk_copy(self.chunk.as_ptr()) }; - let chunk = NonNull::new(chunk).ok_or(MtmdInputChunkError::ChunkOperationFailed)?; + let mut out_chunk: *mut llama_cpp_bindings_sys::mtmd_input_chunk = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_input_chunk_copy( + self.chunk.as_ptr(), + &raw mut out_chunk, + &raw mut out_error, + ) + }; - Ok(Self { chunk, owned: true }) + input_chunk_copy_status_to_result(status, out_chunk, out_error) } /// Checks that this chunk can be evaluated in decodes of `micro_batch_tokens` and returns the @@ -151,31 +219,38 @@ impl Drop for MtmdInputChunk { } #[cfg(test)] -mod unit_tests { - use super::non_causal_chunk_micro_batch_error; +mod tokens_from_raw_ptr_tests { + use std::ptr; + use super::tokens_from_raw_ptr; - use crate::mtmd::mtmd_eval_error::MtmdEvalError; - use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; + use crate::token::LlamaToken; #[test] fn tokens_from_raw_ptr_returns_none_for_null() { - assert!(unsafe { tokens_from_raw_ptr(std::ptr::null(), 5) }.is_none()); + assert_eq!(unsafe { tokens_from_raw_ptr(ptr::null(), 5) }, None); } #[test] fn tokens_from_raw_ptr_returns_none_for_zero_count() { let token: llama_cpp_bindings_sys::llama_token = 42; - assert!(unsafe { tokens_from_raw_ptr(&raw const token, 0) }.is_none()); + + assert_eq!(unsafe { tokens_from_raw_ptr(&raw const token, 0) }, None); } #[test] - fn tokens_from_raw_ptr_returns_some_for_valid() { + fn tokens_from_raw_ptr_views_every_token() { let tokens: [llama_cpp_bindings_sys::llama_token; 2] = [1, 2]; let result = unsafe { tokens_from_raw_ptr(tokens.as_ptr(), 2) }; - assert!(result.is_some()); - assert_eq!(result.unwrap().len(), 2); + assert_eq!(result, Some(&[LlamaToken::new(1), LlamaToken::new(2)][..])); } +} + +#[cfg(test)] +mod non_causal_chunk_micro_batch_error_tests { + use super::non_causal_chunk_micro_batch_error; + use crate::mtmd::mtmd_eval_error::MtmdEvalError; + use crate::mtmd::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; #[test] fn a_non_causal_chunk_larger_than_the_micro_batch_reports_the_mismatch() { @@ -192,11 +267,92 @@ mod unit_tests { #[test] fn a_causal_chunk_larger_than_the_micro_batch_is_split_instead() { - assert!(non_causal_chunk_micro_batch_error(false, 9, 4).is_none()); + assert_eq!(non_causal_chunk_micro_batch_error(false, 9, 4), None); } #[test] fn a_non_causal_chunk_within_the_micro_batch_fits() { - assert!(non_causal_chunk_micro_batch_error(true, 4, 4).is_none()); + assert_eq!(non_causal_chunk_micro_batch_error(true, 4, 4), None); + } +} + +#[cfg(test)] +mod input_chunk_copy_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::ptr; + + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::input_chunk_copy_status_to_result; + use crate::mtmd::mtmd_input_chunk_error::MtmdInputChunkError; + + const OPERATION: &str = "llama_rs_mtmd_input_chunk_copy"; + + fn contract_violation(detail: &'static str) -> MtmdInputChunkError { + FfiContractError { + operation: OPERATION, + detail, + } + .into() + } + + fn missing_message(detail: &'static str) -> MtmdInputChunkError { + CppStringError::Missing(FfiContractError { + operation: OPERATION, + detail, + }) + .into() + } + + #[test] + fn every_failure_status_maps_to_its_error() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_OK, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_CHUNK_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_NULL_OUT_CHUNK_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION, + 255, + ]; + + assert_eq!( + statuses.map(|status| { + input_chunk_copy_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() + }), + [ + Some(contract_violation("success status contained a null chunk")), + Some(contract_violation("was given a null chunk argument")), + Some(contract_violation("was given a null out_chunk argument")), + Some(MtmdInputChunkError::NotEnoughMemory), + Some(MtmdInputChunkError::LlamaCppOutOfMemory), + Some(missing_message( + "reported a thrown C++ exception without an error message" + )), + Some(MtmdInputChunkError::FfiStatus(FfiStatusError { + operation: OPERATION, + code: 255, + })), + ] + ); + } + + #[test] + fn a_thrown_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"allocation threw".as_ptr()) }; + + assert_eq!( + input_chunk_copy_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNK_COPY_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + .err(), + Some(MtmdInputChunkError::Reported { + message: "allocation threw".to_owned(), + }) + ); } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs index ab94f14b3..70a01ba8b 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_error.rs @@ -1,5 +1,19 @@ -#[derive(thiserror::Error, Debug)] +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunkError { - #[error("input chunk operation failed")] - ChunkOperationFailed, + #[error(transparent)] + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_type.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_type.rs index 5392d85e0..5d2473187 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_type.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk_type.rs @@ -25,7 +25,7 @@ impl TryFrom for MtmdInputChunkTy #[cfg(test)] mod tests { - use super::MtmdInputChunkType; + use crate::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; use crate::mtmd::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; #[test] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs index 113ada7f1..10552a150 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks.rs @@ -1,23 +1,80 @@ +use std::ffi::c_char; +use std::ptr; use std::ptr::NonNull; -use crate::context::LlamaContext; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; -use super::micro_batch_tokens::micro_batch_tokens; +use super::eval_chunk::eval_chunk; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; +use super::mtmd_eval_limits::MtmdEvalLimits; use super::mtmd_input_chunk::MtmdInputChunk; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunks_error::MtmdInputChunksError; -use super::positive_batch_tokens::positive_batch_tokens; +use crate::context::LlamaContext; -const fn check_eval_result(result: i32) -> Result<(), MtmdEvalError> { - if result == 0 { - Ok(()) - } else { - Err(MtmdEvalError::EvalFailed { code: result }) +fn input_chunks_init_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_input_chunks_init_status, + out_chunks: *mut llama_cpp_bindings_sys::mtmd_input_chunks, + out_error: *mut c_char, +) -> Result { + const OPERATION: &str = "llama_rs_mtmd_input_chunks_init"; + + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK => NonNull::new(out_chunks) + .map(|chunks| MtmdInputChunks { chunks }) + .ok_or_else(|| { + FfiContractError { + operation: OPERATION, + detail: "success status contained null chunks", + } + .into() + }), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG => { + Err(FfiContractError { + operation: OPERATION, + detail: "was given a null out_chunks argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdInputChunksError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdInputChunksError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdInputChunksError::Reported { message }) + } + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), } } +fn fit_each( + chunks: &[MtmdInputChunk], + mtmd_ctx: &MtmdContext, + micro_batch_tokens: u32, +) -> Result, MtmdEvalError> { + chunks + .iter() + .map(|chunk| chunk.fit_to_micro_batch(mtmd_ctx, micro_batch_tokens)) + .collect() +} + #[derive(Debug)] pub struct MtmdInputChunks { pub chunks: NonNull, @@ -28,14 +85,18 @@ unsafe impl Send for MtmdInputChunks {} impl MtmdInputChunks { /// # Errors /// - /// Returns `MtmdInputChunksError::ChunksCreationFailed` if the underlying llama.cpp function - /// returns null. - /// + /// Returns the [`MtmdInputChunksError`] variant matching the wrapper's status code. pub fn new() -> Result { - let chunks = unsafe { llama_cpp_bindings_sys::mtmd_input_chunks_init() }; - let chunks = NonNull::new(chunks).ok_or(MtmdInputChunksError::ChunksCreationFailed)?; + let mut out_chunks: *mut llama_cpp_bindings_sys::mtmd_input_chunks = ptr::null_mut(); + let mut out_error: *mut c_char = ptr::null_mut(); + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_input_chunks_init( + &raw mut out_chunks, + &raw mut out_error, + ) + }; - Ok(Self { chunks }) + input_chunks_init_status_to_result(status, out_chunks, out_error) } #[must_use] @@ -54,8 +115,9 @@ impl MtmdInputChunks { return None; } - let chunk_ptr = - unsafe { llama_cpp_bindings_sys::mtmd_input_chunks_get(self.chunks.as_ptr(), index) }; + let chunk_ptr = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_input_chunks_get(self.chunks.as_ptr(), index) + }; NonNull::new(chunk_ptr.cast_mut()).map(|ptr| MtmdInputChunk { chunk: ptr, @@ -77,16 +139,7 @@ impl MtmdInputChunks { mtmd_ctx: &MtmdContext, micro_batch_tokens: u32, ) -> Result, MtmdEvalError> { - (0..self.len()) - .map(|index| { - self.get(index) - .ok_or(crate::FfiContractError { - operation: "mtmd_input_chunks_get", - detail: "returned a null chunk within the chunk count", - })? - .fit_to_micro_batch(mtmd_ctx, micro_batch_tokens) - }) - .collect() + fit_each(&self.chunks_within_count()?, mtmd_ctx, micro_batch_tokens) } #[must_use] @@ -104,49 +157,55 @@ impl MtmdInputChunks { /// /// # Errors /// - /// Returns [`MtmdEvalError::BatchSizeExceedsContextLimit`] when `n_batch` exceeds the - /// context's batch size, [`MtmdEvalError::NonPositiveBatchSize`] when it is not positive, - /// any error of [`Self::fit_to_micro_batch`], or [`MtmdEvalError::EvalFailed`] if any - /// encoding or decoding operation fails. + /// Returns any error of [`MtmdEvalLimits::batch_tokens`] or [`Self::fit_to_micro_batch`], or + /// [`MtmdEvalError::EvalFailed`] if any encoding or decoding operation fails. pub fn eval_chunks( &self, mtmd_ctx: &MtmdContext, - llama_ctx: &LlamaContext, + llama_ctx: &mut LlamaContext, start_position: llama_cpp_bindings_sys::llama_pos, seq_id: llama_cpp_bindings_sys::llama_seq_id, n_batch: i32, logits_last: bool, ) -> Result { - let batch_tokens = positive_batch_tokens(n_batch)?; - let context_max_batch = llama_ctx.n_batch(); - - if batch_tokens.get() > context_max_batch { - return Err(MtmdEvalError::BatchSizeExceedsContextLimit { - requested: n_batch, - context_max: context_max_batch, - }); - } - - self.fit_to_micro_batch(mtmd_ctx, micro_batch_tokens(llama_ctx, batch_tokens))?; - - let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; - - let result = unsafe { - llama_cpp_bindings_sys::mtmd_helper_eval_chunks( - mtmd_ctx.context.as_ptr(), - llama_ctx.context.as_ptr(), - self.chunks.as_ptr(), - start_position, - seq_id, - n_batch, - logits_last, - &raw mut final_position, - ) - }; - - check_eval_result(result)?; + let limits = MtmdEvalLimits::of(llama_ctx); + let micro_batch_tokens = limits + .context + .micro_batch_tokens(limits.batch_tokens(n_batch, seq_id)?); + let chunks = self.chunks_within_count()?; + + fit_each(&chunks, mtmd_ctx, micro_batch_tokens)?; + + let chunk_count = chunks.len(); + + chunks + .iter() + .enumerate() + .try_fold(start_position, |position, (index, chunk)| { + eval_chunk( + chunk, + mtmd_ctx, + llama_ctx, + position, + seq_id, + n_batch, + logits_last && index + 1 == chunk_count, + ) + }) + } - Ok(final_position) + fn chunks_within_count(&self) -> Result, MtmdEvalError> { + (0..self.len()) + .map(|index| { + self.get(index).ok_or_else(|| { + FfiContractError { + operation: "mtmd_input_chunks_get", + detail: "returned a null chunk within the chunk count", + } + .into() + }) + }) + .collect() } } @@ -158,7 +217,7 @@ impl Drop for MtmdInputChunks { #[cfg(test)] mod tests { - use super::MtmdInputChunks; + use crate::mtmd::mtmd_input_chunks::MtmdInputChunks; #[test] fn new_creates_empty_chunks() { @@ -175,25 +234,83 @@ mod tests { assert!(chunks.get(0).is_none()); assert!(chunks.get(999).is_none()); } +} - #[test] - fn check_eval_result_ok_for_zero() { - use super::check_eval_result; +#[cfg(test)] +mod input_chunks_init_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::ptr; + + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::input_chunks_init_status_to_result; + use crate::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; - assert!(check_eval_result(0).is_ok()); + const OPERATION: &str = "llama_rs_mtmd_input_chunks_init"; + + fn contract_violation(detail: &'static str) -> MtmdInputChunksError { + FfiContractError { + operation: OPERATION, + detail, + } + .into() + } + + fn missing_message(detail: &'static str) -> MtmdInputChunksError { + CppStringError::Missing(FfiContractError { + operation: OPERATION, + detail, + }) + .into() } #[test] - fn check_eval_result_error_for_nonzero() { - use super::MtmdEvalError; - use super::check_eval_result; + fn every_failure_status_maps_to_its_error() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_OK, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_NULL_OUT_CHUNKS_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_ERROR_STRING_ALLOCATION_FAILED, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_OUT_OF_MEMORY, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + 255, + ]; + + assert_eq!( + statuses.map(|status| { + input_chunks_init_status_to_result(status, ptr::null_mut(), ptr::null_mut()).err() + }), + [ + Some(contract_violation("success status contained null chunks")), + Some(contract_violation("was given a null out_chunks argument")), + Some(MtmdInputChunksError::NotEnoughMemory), + Some(MtmdInputChunksError::LlamaCppOutOfMemory), + Some(missing_message( + "reported a thrown C++ exception without an error message" + )), + Some(MtmdInputChunksError::FfiStatus(FfiStatusError { + operation: OPERATION, + code: 255, + })), + ] + ); + } - let err = check_eval_result(7).unwrap_err(); - let representative = MtmdEvalError::EvalFailed { code: 0 }; + #[test] + fn a_thrown_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"allocation threw".as_ptr()) }; assert_eq!( - std::mem::discriminant(&err), - std::mem::discriminant(&representative) + input_chunks_init_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_INPUT_CHUNKS_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + .err(), + Some(MtmdInputChunksError::Reported { + message: "allocation threw".to_owned(), + }) ); } } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs index 0a8947e0d..45c2bad0f 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunks_error.rs @@ -1,5 +1,19 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdInputChunksError { - #[error("input chunks collection could not be created")] - ChunksCreationFailed, + #[error(transparent)] + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, } diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_part.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_part.rs index 178d23ed7..31a43cadd 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_part.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_part.rs @@ -11,7 +11,7 @@ pub enum MtmdInputPart<'bitmap> { mod tests { use std::thread; - use super::MtmdInputPart; + use crate::mtmd::mtmd_input_part::MtmdInputPart; use crate::mtmd::mtmd_input_text_part::MtmdInputTextPart; #[test] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs new file mode 100644 index 000000000..6c6923cda --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage.rs @@ -0,0 +1,448 @@ +use std::ffi::CStr; +use std::ffi::c_char; +use std::num::NonZeroU32; +use std::ptr; + +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; + +use super::mtmd_context_params::MtmdContextParams; +use super::mtmd_image_token_limit::MtmdImageTokenLimit; +use super::mtmd_memory_usage_error::MtmdMemoryUsageError; +use crate::backend_device_index::backend_device_index; + +const OPERATION: &str = "llama_rs_mtmd_get_memory_usage"; + +fn get_memory_usage_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_get_memory_usage_status, + out_error: *mut c_char, +) -> Result<(), MtmdMemoryUsageError> { + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_MMPROJ_PATH_ARG => { + Err(contract_violation("was given a null mmproj_path argument")) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_ARG => Err( + contract_violation("was given a null out_device_memory argument"), + ), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_COUNT_ARG => { + Err(contract_violation( + "was given a null out_device_memory_count argument", + )) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_IMAGE_MAX_TOKENS_ARG => { + Err(contract_violation( + "was given a null out_image_max_tokens argument", + )) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_USE_NON_CAUSAL_ARG => Err( + contract_violation("was given a null out_use_non_causal argument"), + ), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_DEVICE_MEMORY_CAPACITY_EXCEEDED => { + Err(contract_violation( + "measured memory on more devices than the backend registers", + )) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdMemoryUsageError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdMemoryUsageError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + OPERATION, + "reported a thrown C++ exception without an error message", + ) + }?; + + Err(MtmdMemoryUsageError::Reported { message }) + } + other => Err(FfiStatusError { + operation: OPERATION, + code: i64::from(other), + } + .into()), + } +} + +const fn contract_violation(detail: &'static str) -> MtmdMemoryUsageError { + MtmdMemoryUsageError::FfiContract(FfiContractError { + operation: OPERATION, + detail, + }) +} + +/// The worst-case memory a multimodal projector takes on each backend device, with the image +/// budget it was measured at, as llama.cpp estimates it before the projector is loaded. +#[derive(Debug)] +pub struct MtmdMemoryUsage { + device_memory: Vec, + image_max_tokens: i32, + decodes_images_non_causally: bool, +} + +impl MtmdMemoryUsage { + /// Measures the memory the projector at `mmproj_path` takes when loaded with `params`. + /// + /// # Errors + /// + /// Returns [`MtmdMemoryUsageError::Unloadable`] when llama.cpp cannot load the projector, + /// [`MtmdMemoryUsageError::ContextParams`] when `params` have no C representation, or the + /// variant matching the wrapper's status code. + pub fn measure( + mmproj_path: &CStr, + params: &MtmdContextParams, + ) -> Result { + let context_params = llama_cpp_bindings_sys::mtmd_context_params::try_from(params)?; + let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() }; + let mut device_memory = Vec::with_capacity(device_count); + let mut device_memory_count: usize = 0; + let mut image_max_tokens: i32 = 0; + let mut decodes_images_non_causally = false; + let mut out_error: *mut c_char = ptr::null_mut(); + + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_get_memory_usage( + mmproj_path.as_ptr(), + context_params, + device_memory.as_mut_ptr(), + device_count, + &raw mut device_memory_count, + &raw mut image_max_tokens, + &raw mut decodes_images_non_causally, + &raw mut out_error, + ) + }; + + get_memory_usage_status_to_result(status, out_error)?; + + unsafe { device_memory.set_len(device_memory_count) }; + + if device_memory.is_empty() { + return Err(MtmdMemoryUsageError::Unloadable { + path: mmproj_path.to_owned(), + }); + } + + Ok(Self { + device_memory, + image_max_tokens, + decodes_images_non_causally, + }) + } + + /// Adds the projector's memory to the fit margin of each device it takes memory on, so + /// fitting the text model leaves that memory free, as llama-server does. `margins` is + /// indexed by backend device, like the margins of + /// [`LlamaModelParams::fit_params`](crate::model::params::LlamaModelParams::fit_params); + /// a device past its end has no margin to grow. + pub fn add_to_fit_margins(&self, margins: &mut [usize]) { + for llama_cpp_bindings_sys::llama_rs_mtmd_device_memory { device, bytes } in + &self.device_memory + { + let margin = backend_device_index(*device) + .and_then(|device_index| margins.get_mut(device_index)); + + if let Some(margin) = margin { + *margin += bytes; + } + } + } + + /// Lowers the image token limits of `params` to `micro_batch_tokens` when the projector + /// decodes images non-causally and could produce more tokens than one micro-batch holds, + /// because a non-causal image must be decoded in a single micro-batch. + #[must_use] + pub fn fit_image_tokens_into_micro_batch( + &self, + params: MtmdContextParams, + micro_batch_tokens: NonZeroU32, + ) -> MtmdContextParams { + if !self.decodes_images_non_causally + || i64::from(self.image_max_tokens) <= i64::from(micro_batch_tokens.get()) + { + return params; + } + + MtmdContextParams { + image_min_tokens: params.image_min_tokens.at_most(micro_batch_tokens), + image_max_tokens: MtmdImageTokenLimit::Tokens(micro_batch_tokens), + ..params + } + } +} + +#[cfg(test)] +mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::num::NonZeroU32; + use std::ptr; + + use llama_cpp_fault_injection::armed_fault::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::get_memory_usage_status_to_result; + use crate::mtmd::mtmd_context_params::MtmdContextParams; + use crate::mtmd::mtmd_context_params_error::MtmdContextParamsError; + use crate::mtmd::mtmd_image_token_limit::MtmdImageTokenLimit; + use crate::mtmd::mtmd_memory_usage::MtmdMemoryUsage; + use crate::mtmd::mtmd_memory_usage_error::MtmdMemoryUsageError; + + const MICRO_BATCH_TOKENS: NonZeroU32 = NonZeroU32::new(64).unwrap(); + + fn first_device_usage(bytes: usize) -> MtmdMemoryUsage { + MtmdMemoryUsage { + device_memory: vec![llama_cpp_bindings_sys::llama_rs_mtmd_device_memory { + device: unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(0) }, + bytes, + }], + image_max_tokens: -1, + decodes_images_non_causally: false, + } + } + + fn image_usage(image_max_tokens: i32, decodes_images_non_causally: bool) -> MtmdMemoryUsage { + MtmdMemoryUsage { + device_memory: Vec::new(), + image_max_tokens, + decodes_images_non_causally, + } + } + + fn params_with_min_tokens(image_min_tokens: MtmdImageTokenLimit) -> MtmdContextParams { + MtmdContextParams { + image_min_tokens, + ..MtmdContextParams::default() + } + } + + fn contract_violation(detail: &'static str) -> MtmdMemoryUsageError { + FfiContractError { + operation: "llama_rs_mtmd_get_memory_usage", + detail, + } + .into() + } + + fn missing_message(detail: &'static str) -> MtmdMemoryUsageError { + CppStringError::Missing(FfiContractError { + operation: "llama_rs_mtmd_get_memory_usage", + detail, + }) + .into() + } + + #[test] + fn measuring_a_missing_projector_reports_it_unloadable() { + assert_eq!( + MtmdMemoryUsage::measure(c"/nonexistent/mmproj.gguf", &MtmdContextParams::default()) + .err(), + Some(MtmdMemoryUsageError::Unloadable { + path: c"/nonexistent/mmproj.gguf".to_owned(), + }) + ); + } + + #[test] + fn a_measurement_llama_cpp_fails_is_reported() { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_MTMD_GET_MEMORY_USAGE_THROWS, + ); + + assert_eq!( + MtmdMemoryUsage::measure(c"/nonexistent/mmproj.gguf", &MtmdContextParams::default()) + .err(), + Some(MtmdMemoryUsageError::Reported { + message: injected_fault_message(), + }) + ); + } + + #[test] + fn measuring_with_unrepresentable_params_is_refused() { + let params = MtmdContextParams { + image_max_tokens: MtmdImageTokenLimit::Tokens(NonZeroU32::MAX), + ..MtmdContextParams::default() + }; + + assert_eq!( + MtmdMemoryUsage::measure(c"/nonexistent/mmproj.gguf", ¶ms).err(), + i32::try_from(u32::MAX).err().map(|source| { + MtmdMemoryUsageError::ContextParams( + MtmdContextParamsError::ImageMaxTokensExceedCInt { + limit: MtmdImageTokenLimit::Tokens(NonZeroU32::MAX), + source, + }, + ) + }) + ); + } + + #[test] + fn projector_memory_grows_the_margin_of_its_device() { + let mut margins = [1, 2]; + + first_device_usage(5).add_to_fit_margins(&mut margins); + + assert_eq!(margins, [6, 2]); + } + + #[test] + fn projector_memory_on_a_device_without_a_margin_is_not_added() { + let mut margins: [usize; 0] = []; + + first_device_usage(5).add_to_fit_margins(&mut margins); + + assert_eq!(margins, [0_usize; 0]); + } + + #[test] + fn a_causal_projector_keeps_the_image_token_limits() { + let params = params_with_min_tokens(MtmdImageTokenLimit::Tokens(NonZeroU32::MAX)); + + assert_eq!( + image_usage(4096, false) + .fit_image_tokens_into_micro_batch(params.clone(), MICRO_BATCH_TOKENS), + params + ); + } + + #[test] + fn a_non_causal_projector_within_the_micro_batch_keeps_the_image_token_limits() { + let params = params_with_min_tokens(MtmdImageTokenLimit::Tokens(NonZeroU32::MAX)); + + assert_eq!( + image_usage(64, true) + .fit_image_tokens_into_micro_batch(params.clone(), MICRO_BATCH_TOKENS), + params + ); + } + + #[test] + fn a_non_causal_projector_beyond_the_micro_batch_lowers_the_image_token_limits_to_it() { + let params = params_with_min_tokens(MtmdImageTokenLimit::Tokens(NonZeroU32::MAX)); + + assert_eq!( + image_usage(65, true) + .fit_image_tokens_into_micro_batch(params.clone(), MICRO_BATCH_TOKENS), + MtmdContextParams { + image_min_tokens: MtmdImageTokenLimit::Tokens(MICRO_BATCH_TOKENS), + image_max_tokens: MtmdImageTokenLimit::Tokens(MICRO_BATCH_TOKENS), + ..params + } + ); + } + + #[test] + fn the_ok_status_is_success() { + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_OK, + ptr::null_mut(), + ), + Ok(()) + ); + } + + #[test] + fn wrapper_contract_statuses_are_contract_violations() { + let statuses = [ + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_MMPROJ_PATH_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_DEVICE_MEMORY_COUNT_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_IMAGE_MAX_TOKENS_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_NULL_OUT_USE_NON_CAUSAL_ARG, + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_DEVICE_MEMORY_CAPACITY_EXCEEDED, + ]; + + assert_eq!( + statuses.map(|status| get_memory_usage_status_to_result(status, ptr::null_mut())), + [ + Err(contract_violation("was given a null mmproj_path argument")), + Err(contract_violation( + "was given a null out_device_memory argument" + )), + Err(contract_violation( + "was given a null out_device_memory_count argument" + )), + Err(contract_violation( + "was given a null out_image_max_tokens argument" + )), + Err(contract_violation( + "was given a null out_use_non_causal argument" + )), + Err(contract_violation( + "measured memory on more devices than the backend registers" + )), + ] + ); + } + + #[test] + fn a_failed_error_string_allocation_is_not_enough_memory() { + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut(), + ), + Err(MtmdMemoryUsageError::NotEnoughMemory) + ); + } + + #[test] + fn llama_cpp_running_out_of_memory_is_reported() { + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut(), + ), + Err(MtmdMemoryUsageError::LlamaCppOutOfMemory) + ); + } + + #[test] + fn a_thrown_exception_reports_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"projector threw".as_ptr()) }; + + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(MtmdMemoryUsageError::Reported { + message: "projector threw".to_owned(), + }) + ); + } + + #[test] + fn a_thrown_exception_without_a_message_is_a_contract_violation() { + assert_eq!( + get_memory_usage_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_GET_MEMORY_USAGE_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ), + Err(missing_message( + "reported a thrown C++ exception without an error message" + )) + ); + } + + #[test] + fn an_unknown_status_is_preserved() { + assert_eq!( + get_memory_usage_status_to_result(255, ptr::null_mut()), + Err(MtmdMemoryUsageError::FfiStatus(FfiStatusError { + operation: "llama_rs_mtmd_get_memory_usage", + code: 255, + })) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs new file mode 100644 index 000000000..fce538d5e --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/mtmd_memory_usage_error.rs @@ -0,0 +1,27 @@ +use std::ffi::CString; + +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + +use super::mtmd_context_params_error::MtmdContextParamsError; + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum MtmdMemoryUsageError { + #[error(transparent)] + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), + #[error(transparent)] + FfiContract(#[from] FfiContractError), + #[error(transparent)] + ContextParams(#[from] MtmdContextParamsError), + #[error("the memory usage of mmproj {path:?} could not be measured, because it does not load")] + Unloadable { path: CString }, + #[error("not enough memory")] + NotEnoughMemory, + #[error("the llama.cpp library ran out of memory")] + LlamaCppOutOfMemory, + #[error("{message}")] + Reported { message: String }, +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs index 29c36b004..9f90570c8 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_tokenize_error.rs @@ -1,11 +1,17 @@ +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use crate::mtmd::mtmd_input_chunks_error::MtmdInputChunksError; #[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum MtmdTokenizeError { #[error(transparent)] - FfiStatus(#[from] crate::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] crate::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("Failed to create CString from input text: {0}")] CStringError(#[from] std::ffi::NulError), #[error("{0}")] diff --git a/llama-cpp-bindings/src/mtmd/optional_mtmd_id.rs b/llama-cpp-bindings/src/mtmd/optional_mtmd_id.rs new file mode 100644 index 000000000..ffb1ca4f9 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/optional_mtmd_id.rs @@ -0,0 +1,50 @@ +use std::ffi::CStr; +use std::ffi::c_char; +use std::str::Utf8Error; + +/// The id mtmd reports for a bitmap or a chunk, or `None` when mtmd reports no id. +/// +/// # Errors +/// +/// Returns [`Utf8Error`] when the id is not valid UTF-8. +/// +/// # Safety +/// +/// `id` must be null or point to a NUL-terminated string that outlives the call. +pub unsafe fn optional_mtmd_id(id: *const c_char) -> Result, Utf8Error> { + if id.is_null() { + return Ok(None); + } + + Ok(Some(unsafe { CStr::from_ptr(id) }.to_str()?.to_owned())) +} + +#[cfg(test)] +mod tests { + use std::ptr; + + use super::optional_mtmd_id; + + #[test] + fn a_missing_id_is_none() { + assert_eq!(unsafe { optional_mtmd_id(ptr::null()) }, Ok(None)); + } + + #[test] + fn an_id_is_copied() { + assert_eq!( + unsafe { optional_mtmd_id(c"image".as_ptr()) }, + Ok(Some("image".to_owned())) + ); + } + + #[test] + fn an_id_that_is_not_utf8_is_refused() { + let not_utf8 = c"\xff"; + + assert_eq!( + unsafe { optional_mtmd_id(not_utf8.as_ptr()) }, + Err(not_utf8.to_str().expect_err("0xff is not UTF-8")) + ); + } +} diff --git a/llama-cpp-bindings/src/mtmd/positive_batch_tokens.rs b/llama-cpp-bindings/src/mtmd/positive_batch_tokens.rs deleted file mode 100644 index 060aa6bd1..000000000 --- a/llama-cpp-bindings/src/mtmd/positive_batch_tokens.rs +++ /dev/null @@ -1,37 +0,0 @@ -use std::num::NonZeroU32; - -use super::mtmd_eval_error::MtmdEvalError; - -/// Returns `n_batch` as a positive token count. -/// -/// # Errors -/// -/// Returns [`MtmdEvalError::NonPositiveBatchSize`] when `n_batch` is not positive. -pub fn positive_batch_tokens(n_batch: i32) -> Result { - u32::try_from(n_batch) - .ok() - .and_then(NonZeroU32::new) - .ok_or(MtmdEvalError::NonPositiveBatchSize { requested: n_batch }) -} - -#[cfg(test)] -mod tests { - use super::positive_batch_tokens; - use crate::mtmd::mtmd_eval_error::MtmdEvalError; - - #[test] - fn a_zero_batch_is_rejected() { - assert_eq!( - positive_batch_tokens(0), - Err(MtmdEvalError::NonPositiveBatchSize { requested: 0 }) - ); - } - - #[test] - fn a_negative_batch_is_rejected() { - assert_eq!( - positive_batch_tokens(-1), - Err(MtmdEvalError::NonPositiveBatchSize { requested: -1 }) - ); - } -} diff --git a/llama-cpp-bindings/src/parsed_chat.rs b/llama-cpp-bindings/src/parsed_chat.rs new file mode 100644 index 000000000..897c09f24 --- /dev/null +++ b/llama-cpp-bindings/src/parsed_chat.rs @@ -0,0 +1,117 @@ +use std::ffi::c_char; +use std::ptr::NonNull; + +use llama_cpp_bindings_types::model_provided_tool_call_id::ModelProvidedToolCallId; +use llama_cpp_bindings_types::model_provided_tool_call_id_error::ModelProvidedToolCallIdError; +use llama_cpp_bindings_types::parsed_chat_message::ParsedChatMessage; +use llama_cpp_bindings_types::parsed_tool_call::ParsedToolCall; +use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; +use llama_cpp_bindings_types::tool_call_arguments::ToolCallArguments; + +use crate::cpp_text_view::cpp_text_view; +use crate::error::parse_chat_message_error::ParseChatMessageError; + +fn parsed_text( + read: impl FnOnce(*mut usize) -> *const c_char, +) -> Result { + let mut length = 0; + let text = read(&raw mut length); + + unsafe { cpp_text_view(text, length) } + .map(str::to_owned) + .map_err(ParseChatMessageError::ParsedTextNotUtf8) +} + +fn tool_call_id_reported_by_llama_cpp(llama_cpp_tool_call_id: String) -> ParsedToolCallId { + match ModelProvidedToolCallId::try_from(llama_cpp_tool_call_id) { + Ok(model_provided_tool_call_id) => { + ParsedToolCallId::ModelProvided(model_provided_tool_call_id) + } + Err(ModelProvidedToolCallIdError::Empty) => ParsedToolCallId::NotProvidedByModel, + } +} + +fn parsed_tool_call( + handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, + index: usize, +) -> Result { + let id = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_id(handle, index, out_length) + })?; + let name = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_name(handle, index, out_length) + })?; + let arguments = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_arguments(handle, index, out_length) + })?; + + Ok(ParsedToolCall::new( + tool_call_id_reported_by_llama_cpp(id), + name, + ToolCallArguments::from_string(arguments), + )) +} + +fn collect_parsed_chat_message( + handle: *mut llama_cpp_bindings_sys::llama_rs_parsed_chat, +) -> Result { + let content = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_content(handle, out_length) + })?; + let reasoning_content = parsed_text(|out_length| unsafe { + llama_cpp_bindings_sys::llama_rs_parsed_chat_reasoning_content(handle, out_length) + })?; + let tool_call_count = + unsafe { llama_cpp_bindings_sys::llama_rs_parsed_chat_tool_call_count(handle) }; + let tool_calls = (0..tool_call_count) + .map(|index| parsed_tool_call(handle, index)) + .collect::, ParseChatMessageError>>()?; + + Ok(ParsedChatMessage::new( + content, + reasoning_content, + tool_calls, + )) +} + +/// Reads the message llama.cpp parsed into `handle` and releases the handle. +/// +/// # Errors +/// +/// Returns [`ParseChatMessageError`] when an accessor fails or reports text that is not UTF-8, +/// or when releasing the handle fails. +pub fn take_parsed_chat_message( + handle: NonNull, +) -> Result { + let message = collect_parsed_chat_message(handle.as_ptr()); + + unsafe { llama_cpp_bindings_sys::llama_rs_parsed_chat_free(handle.as_ptr()) }; + + message +} + +#[cfg(test)] +mod tests { + + use llama_cpp_bindings_types::model_provided_tool_call_id::ModelProvidedToolCallId; + use llama_cpp_bindings_types::parsed_tool_call_id::ParsedToolCallId; + + use super::tool_call_id_reported_by_llama_cpp; + + #[test] + fn an_empty_llama_cpp_tool_call_id_is_not_provided_by_the_model() { + assert_eq!( + tool_call_id_reported_by_llama_cpp(String::new()), + ParsedToolCallId::NotProvidedByModel + ); + } + + #[test] + fn a_non_empty_llama_cpp_tool_call_id_is_provided_by_the_model() { + assert_eq!( + Ok(tool_call_id_reported_by_llama_cpp("abc123def".to_owned())), + ModelProvidedToolCallId::try_from("abc123def".to_owned()) + .map(ParsedToolCallId::ModelProvided) + ); + } +} diff --git a/llama-cpp-bindings/src/pending_chat_piece.rs b/llama-cpp-bindings/src/pending_chat_piece.rs new file mode 100644 index 000000000..d5e3219a0 --- /dev/null +++ b/llama-cpp-bindings/src/pending_chat_piece.rs @@ -0,0 +1,10 @@ +use std::ops::Range; + +use crate::token::LlamaToken; + +#[derive(Debug)] +pub struct PendingChatPiece { + pub token: LlamaToken, + pub bytes: Range, + pub piece: String, +} diff --git a/llama-cpp-bindings/src/raw_chat_message.rs b/llama-cpp-bindings/src/raw_chat_message.rs index 15b2f2662..1d2bfe4d6 100644 --- a/llama-cpp-bindings/src/raw_chat_message.rs +++ b/llama-cpp-bindings/src/raw_chat_message.rs @@ -1,24 +1,8 @@ +use crate::chat_output_completeness::ChatOutputCompleteness; + #[derive(Debug, Eq, PartialEq)] pub struct RawChatMessage { pub text: String, - pub is_partial: bool, + pub completeness: ChatOutputCompleteness, pub ffi_error_message: String, } - -#[cfg(test)] -mod tests { - use super::RawChatMessage; - - #[test] - fn carries_text_partial_flag_and_ffi_error_message() { - let raw = RawChatMessage { - text: "hello".to_owned(), - is_partial: true, - ffi_error_message: "parser bailed".to_owned(), - }; - - assert_eq!(raw.text, "hello"); - assert!(raw.is_partial); - assert_eq!(raw.ffi_error_message, "parser bailed"); - } -} diff --git a/llama-cpp-bindings/src/resolved_tool_call_markers.rs b/llama-cpp-bindings/src/resolved_tool_call_markers.rs deleted file mode 100644 index 3b2e6a3c4..000000000 --- a/llama-cpp-bindings/src/resolved_tool_call_markers.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ResolvedToolCallMarkers { - pub open: Option, - pub close: Option, -} diff --git a/llama-cpp-bindings/src/sampled_token.rs b/llama-cpp-bindings/src/sampled_token.rs index 776ead808..b93e50573 100644 --- a/llama-cpp-bindings/src/sampled_token.rs +++ b/llama-cpp-bindings/src/sampled_token.rs @@ -1,3 +1,4 @@ +use crate::sampled_token_section::SampledTokenSection; use crate::token::LlamaToken; #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] @@ -7,3 +8,82 @@ pub enum SampledToken { ToolCall(LlamaToken), Undeterminable(LlamaToken), } + +impl SampledToken { + #[must_use] + pub const fn in_section(section: SampledTokenSection, token: LlamaToken) -> Self { + match section { + SampledTokenSection::Reasoning => Self::Reasoning(token), + SampledTokenSection::Content => Self::Content(token), + SampledTokenSection::ToolCall => Self::ToolCall(token), + SampledTokenSection::Pending => Self::Undeterminable(token), + } + } + + #[must_use] + pub const fn section(&self) -> SampledTokenSection { + match self { + Self::Reasoning(_) => SampledTokenSection::Reasoning, + Self::Content(_) => SampledTokenSection::Content, + Self::ToolCall(_) => SampledTokenSection::ToolCall, + Self::Undeterminable(_) => SampledTokenSection::Pending, + } + } + + #[must_use] + pub const fn token(&self) -> LlamaToken { + match *self { + Self::Content(token) + | Self::Reasoning(token) + | Self::ToolCall(token) + | Self::Undeterminable(token) => token, + } + } +} + +#[cfg(test)] +mod tests { + use crate::sampled_token::SampledToken; + use crate::sampled_token_section::SampledTokenSection; + use crate::token::LlamaToken; + + #[test] + fn every_section_carries_the_sampled_token() { + let token = LlamaToken::new(42); + + assert_eq!( + [ + SampledToken::Content(token), + SampledToken::Reasoning(token), + SampledToken::ToolCall(token), + SampledToken::Undeterminable(token), + ] + .map(|sampled_token| sampled_token.token()), + [token; 4] + ); + } + + #[test] + fn a_token_placed_in_a_section_reports_that_section() { + let token = LlamaToken::new(7); + let sections = [ + SampledTokenSection::Pending, + SampledTokenSection::Content, + SampledTokenSection::Reasoning, + SampledTokenSection::ToolCall, + ]; + + assert_eq!( + sections.map(|section| SampledToken::in_section(section, token).section()), + sections + ); + } + + #[test] + fn a_token_of_the_pending_section_is_undeterminable() { + assert_eq!( + SampledToken::in_section(SampledTokenSection::Pending, LlamaToken::new(7)), + SampledToken::Undeterminable(LlamaToken::new(7)) + ); + } +} diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index 59fa49a75..749538115 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -1,357 +1,192 @@ use std::collections::VecDeque; -use std::sync::Arc; use llama_cpp_bindings_sys::llama_pos; use llama_cpp_bindings_sys::llama_seq_id; +use llama_cpp_bindings_types::token_usage::TokenUsage; +use llama_cpp_bindings_types::token_usage_error::TokenUsageError; -use llama_cpp_bindings_types::TokenUsage; -use llama_cpp_bindings_types::TokenUsageError; - -use crate::bare_json_tool_calls::BareJsonToolCalls; use crate::batch_add_error::BatchAddError; +use crate::chat_output::ChatOutput; +use crate::chat_output_completeness::ChatOutputCompleteness; +use crate::chat_piece_role::ChatPieceRole; +use crate::chat_session::ChatSession; +use crate::chat_session_attribution::ChatSessionAttribution; +use crate::classified_sample::ClassifiedSample; use crate::context::LlamaContext; -use crate::error::EvalMultimodalChunksError; -use crate::error::SampleError; -use crate::error::TokenToStringError; +use crate::error::eval_multimodal_chunks_error::EvalMultimodalChunksError; +use crate::error::ingest_error::IngestError; +use crate::error::sample_error::SampleError; use crate::eval_multimodal_chunks_params::EvalMultimodalChunksParams; use crate::generation_progress::GenerationProgress; -use crate::json_probe_outcome::JsonProbeOutcome; +use crate::ingest_outcome::IngestOutcome; use crate::llama_batch::LlamaBatch; use crate::model::LlamaModel; -use crate::mtmd::FittedMtmdInputChunk; -use crate::mtmd::FittedMtmdInputChunks; -use crate::mtmd::MtmdContext; -use crate::mtmd::MtmdInputChunkType; -use crate::mtmd::MtmdInputChunks; -use crate::mtmd::micro_batch_tokens; -use crate::mtmd::positive_batch_tokens::positive_batch_tokens; +use crate::mtmd::fitted_mtmd_input_chunk::FittedMtmdInputChunk; +use crate::mtmd::fitted_mtmd_input_chunks::FittedMtmdInputChunks; +use crate::mtmd::mtmd_context::MtmdContext; +use crate::mtmd::mtmd_eval_limits::MtmdEvalLimits; +use crate::mtmd::mtmd_input_chunk_type::MtmdInputChunkType; +use crate::mtmd::mtmd_input_chunks::MtmdInputChunks; +use crate::pending_chat_piece::PendingChatPiece; use crate::sampled_token::SampledToken; +use crate::sampled_token_section::SampledTokenSection; use crate::sampling::LlamaSampler; -use crate::streaming_json_probe::StreamingJsonProbe; -use crate::streaming_markers::StreamingMarkers; use crate::token::LlamaToken; +use crate::token_attribution::TokenAttribution; use crate::token_piece::TokenPiece; -pub use crate::classified_sample::ClassifiedSample; -use crate::ingest_outcome::IngestOutcome; -pub use crate::sampled_token_section::SampledTokenSection; - -#[derive(Copy, Clone, Debug)] -enum PendingMarkerStatus { - Unmatched, - ResolvedBoundary, - AmbiguousBoundary, -} - -#[derive(Clone, Debug)] -struct PendingToken { - token: LlamaToken, - decoded: String, - section: SampledTokenSection, - section_before_token: SampledTokenSection, - marker_status: PendingMarkerStatus, - is_from_prompt: bool, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -enum ProbeMode { - Idle, - Active { - probe: StreamingJsonProbe, - held_count: usize, - }, +const fn outcome_of(token: LlamaToken, role: ChatPieceRole, piece: String) -> IngestOutcome { + IngestOutcome { + sampled_token: SampledToken::in_section(role.section(), token), + piece: match role { + ChatPieceRole::Text(_) => TokenPiece::Visible(piece), + ChatPieceRole::Marker(_) => TokenPiece::Marker(piece), + }, + } } -pub struct SampledTokenClassifier<'model> { +/// Attributes every generated token to reasoning, content, or a tool call, the way llama.cpp's +/// parse of the chat template attributes the text the token decoded to, and counts the tokens +/// of each section. +pub struct SampledTokenClassifier<'session, 'model> { model: &'model LlamaModel, - markers: Arc, - marker_lookback: usize, - bare_json_tool_calls: BareJsonToolCalls, + attribution: TokenAttribution<'session, 'model>, decoder: encoding_rs::Decoder, - pending: VecDeque, section: SampledTokenSection, pending_prompt_tokens: u64, usage: TokenUsage, - probe_mode: ProbeMode, } -impl<'model> SampledTokenClassifier<'model> { +impl<'session, 'model> SampledTokenClassifier<'session, 'model> { + /// A classifier for generation outside of a chat template, whose tokens belong to no + /// section. #[must_use] - pub fn new( - model: &'model LlamaModel, - markers: Arc, - bare_json_tool_calls: BareJsonToolCalls, - ) -> Self { + pub fn undeterminable(model: &'model LlamaModel) -> Self { Self { model, - marker_lookback: markers.max_token_len().saturating_sub(1), - markers, - bare_json_tool_calls, + attribution: TokenAttribution::Undeterminable, + decoder: encoding_rs::UTF_8.new_decoder(), + section: SampledTokenSection::Pending, + pending_prompt_tokens: 0, + usage: TokenUsage::new(), + } + } + + /// A classifier for the answer to `session`. + #[must_use] + pub fn for_chat_session(session: &'session ChatSession<'model>) -> Self { + Self { + model: session.model(), + attribution: TokenAttribution::ChatSession(ChatSessionAttribution { + session, + preserved_tokens: session.preserved_tokens(), + output: ChatOutput::default(), + pending_pieces: VecDeque::new(), + }), decoder: encoding_rs::UTF_8.new_decoder(), - pending: VecDeque::new(), section: SampledTokenSection::Pending, pending_prompt_tokens: 0, usage: TokenUsage::new(), - probe_mode: ProbeMode::Idle, } } - /// Classifies a generated token and appends every outcome it finalises to - /// `outcomes`. An end-of-generation token is counted in the current section - /// without being detokenised, and releases every token still held back. + /// Classifies a generated token and appends every outcome it settles to `outcomes`. An + /// end-of-generation token settles every token still pending and is counted in the section + /// generation ended in. /// /// # Errors - /// Returns [`TokenToStringError`] when the sampled token cannot be - /// detokenised. The failure is surfaced rather than substituting an empty - /// piece, so classification never silently drops generated text. + /// + /// Returns [`IngestError`] when the token cannot be detokenized or llama.cpp fails while + /// parsing the answer. pub fn ingest( &mut self, token: LlamaToken, outcomes: &mut Vec, - ) -> Result { + ) -> Result { if self.model.is_eog_token(&SampledToken::Content(token)) { + self.finish(outcomes)?; self.record_usage_in(self.section); - self.finish(outcomes); return Ok(GenerationProgress::Ended); } - let decoded = self.decode(token)?; - - if self.markers.is_empty() { - self.usage.record_undeterminable_token(); - outcomes.push(IngestOutcome { - sampled_token: SampledToken::Undeterminable(token), - piece: TokenPiece::Visible(decoded), - }); - - return Ok(GenerationProgress::Continues); - } - - self.pending.push_back(PendingToken { - token, - decoded, - section: self.section, - section_before_token: self.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - - self.try_consume_marker_at_tail(); - self.classify_pending_tail(outcomes); - self.drain_overflow(outcomes); - - Ok(GenerationProgress::Continues) - } - - fn classify_pending_tail(&mut self, outcomes: &mut Vec) { - let Some(tail) = self.pending.back() else { - return; - }; - let section_disengages_probe = self.section_disengages_probe(); - let probe_engages = - matches!(self.probe_mode, ProbeMode::Idle) && self.probe_engages_on(&tail.decoded); - - let probe_outcome = match &mut self.probe_mode { - ProbeMode::Active { .. } if section_disengages_probe => { - self.abandon_probe_held_before_tail(outcomes); - - return; - } - ProbeMode::Active { probe, held_count } => { - *held_count += 1; - - probe.feed(&tail.decoded) + let attribution = match &mut self.attribution { + TokenAttribution::Undeterminable => { + let piece = self.model.token_to_piece( + &SampledToken::Content(token), + &mut self.decoder, + true, + None, + )?; + self.usage.record_undeterminable_token(); + outcomes.push(IngestOutcome { + sampled_token: SampledToken::Undeterminable(token), + piece: TokenPiece::Visible(piece), + }); + + return Ok(GenerationProgress::Continues); } - ProbeMode::Idle if probe_engages => { - let mut probe = StreamingJsonProbe::default(); - let probe_outcome = probe.feed(&tail.decoded); - - self.probe_mode = ProbeMode::Active { - probe, - held_count: 1, - }; - - probe_outcome - } - ProbeMode::Idle => return, + TokenAttribution::ChatSession(attribution) => attribution, }; + let renders_special = attribution.preserved_tokens.contains(&token); + let piece = self.model.token_to_piece( + &SampledToken::Content(token), + &mut self.decoder, + renders_special, + None, + )?; - match probe_outcome { - JsonProbeOutcome::StillPossiblyValid => {} - JsonProbeOutcome::CompletedValid => self.commit_probe_as_tool_call(outcomes), - JsonProbeOutcome::Failed => self.abandon_probe(outcomes), - } - } - - const fn section_disengages_probe(&self) -> bool { - matches!( - self.section, - SampledTokenSection::ToolCall | SampledTokenSection::Reasoning - ) - } - - fn probe_engages_on(&self, piece: &str) -> bool { - self.bare_json_tool_calls == BareJsonToolCalls::Detect - && matches!( - self.section, - SampledTokenSection::Content | SampledTokenSection::Pending - ) - && piece.trim_start().starts_with('{') - } - - pub fn ingest_prompt_token(&mut self, token: LlamaToken) { - if self.markers.is_empty() { - return; - } - - if self.model.is_eog_token(&SampledToken::Content(token)) { - self.pending.clear(); - self.section = SampledTokenSection::Pending; - - return; - } + let bytes = attribution.output.push(token, &piece); - self.pending.push_back(PendingToken { + attribution.pending_pieces.push_back(PendingChatPiece { token, - decoded: String::new(), - section: self.section, - section_before_token: self.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: true, + bytes, + piece, }); + self.attribute(ChatOutputCompleteness::Partial, outcomes)?; - self.try_consume_marker_at_tail(); - self.drain_overflow(&mut Vec::new()); - } - - pub fn ingest_prompt_tokens(&mut self, tokens: &[LlamaToken]) { - if self.markers.is_empty() { - return; - } - for &token in tokens { - self.ingest_prompt_token(token); - } - } - - /// Releases every generated token still held back, for a generation that - /// stops without an end-of-generation token (a token limit or a stop request). - pub fn finish(&mut self, outcomes: &mut Vec) { - self.probe_mode = ProbeMode::Idle; - - while let Some(entry) = self.pending.pop_front() { - if !entry.is_from_prompt { - outcomes.push(self.finalize_entry(entry)); - } - } - } - - fn decode(&mut self, token: LlamaToken) -> Result { - self.model - .token_to_piece(&SampledToken::Content(token), &mut self.decoder, true, None) - } - - fn try_consume_marker_at_tail(&mut self) { - let Some(marker) = self - .markers - .longest_matching_suffix(&self.pending.iter().rev().map(|entry| entry.token)) - else { - return; - }; - let span_start = self.pending.len() - marker.tokens().len(); - let span_section = marker.span_section(self.pending[span_start].section_before_token); - let next_section = marker.next_section(); - let is_ambiguous_prefix = self.markers.is_prefix_of_longer_marker(marker.tokens()); - - self.mark_marker_span(span_start, span_section, next_section, is_ambiguous_prefix); + Ok(GenerationProgress::Continues) } - fn mark_marker_span( + fn attribute( &mut self, - span_start: usize, - span_section: SampledTokenSection, - next_section: SampledTokenSection, - is_ambiguous_prefix: bool, - ) { - let marker_status = if is_ambiguous_prefix { - PendingMarkerStatus::AmbiguousBoundary - } else { - PendingMarkerStatus::ResolvedBoundary + completeness: ChatOutputCompleteness, + outcomes: &mut Vec, + ) -> Result<(), IngestError> { + let TokenAttribution::ChatSession(attribution) = &mut self.attribution else { + return Ok(()); }; - for entry in self.pending.iter_mut().skip(span_start) { - entry.marker_status = marker_status; - entry.section = span_section; - } - - self.section = next_section; - } - - const fn probe_held_count(&self) -> usize { - match self.probe_mode { - ProbeMode::Active { held_count, .. } => held_count, - ProbeMode::Idle => 0, - } - } - - fn drain_overflow(&mut self, outcomes: &mut Vec) { - let probe_held_count = self.probe_held_count(); - - while let Some(front) = self.pending.front() { - let drainable = self.pending.len().saturating_sub(probe_held_count); - - if drainable == 0 { - break; - } - - let beyond_lookback = drainable > self.marker_lookback; - let resolved_boundary = - matches!(front.marker_status, PendingMarkerStatus::ResolvedBoundary); - - if !resolved_boundary && !beyond_lookback { - break; - } - - let Some(entry) = self.pending.pop_front() else { + let classification = attribution + .session + .classify(&attribution.output, completeness)?; + let mut released = Vec::new(); + + while let Some(pending_piece) = attribution.pending_pieces.pop_front() { + let Some(role) = classification.classify_token(&pending_piece.bytes) else { + attribution.pending_pieces.push_front(pending_piece); break; }; - if !entry.is_from_prompt { - outcomes.push(self.finalize_entry(entry)); - } + released.push(outcome_of(pending_piece.token, role, pending_piece.piece)); } - } - - fn take_probe_held_tokens(&mut self) -> VecDeque { - let probe_held_count = self.probe_held_count(); - - self.probe_mode = ProbeMode::Idle; - self.pending - .split_off(self.pending.len().saturating_sub(probe_held_count)) - } - - fn commit_probe_as_tool_call(&mut self, outcomes: &mut Vec) { - self.section = SampledTokenSection::Content; - - for mut entry in self.take_probe_held_tokens() { - entry.section = SampledTokenSection::ToolCall; - outcomes.push(self.finalize_entry(entry)); + for outcome in released { + self.section = outcome.sampled_token.section(); + self.record_usage_in(self.section); + outcomes.push(outcome); } - } - fn abandon_probe(&mut self, outcomes: &mut Vec) { - for entry in self.take_probe_held_tokens() { - outcomes.push(self.finalize_entry(entry)); - } + Ok(()) } - fn abandon_probe_held_before_tail(&mut self, outcomes: &mut Vec) { - let Some(tail) = self.pending.pop_back() else { - return; - }; - - self.abandon_probe(outcomes); - self.pending.push_back(tail); + /// Settles every generated token still pending, for a generation that stops without an + /// end-of-generation token (a token limit or a stop request). + /// + /// # Errors + /// + /// Returns [`IngestError`] when llama.cpp fails while parsing the finished answer. + pub fn finish(&mut self, outcomes: &mut Vec) -> Result<(), IngestError> { + self.attribute(ChatOutputCompleteness::Finished, outcomes) } const fn record_usage_in(&mut self, section: SampledTokenSection) { @@ -363,34 +198,10 @@ impl<'model> SampledTokenClassifier<'model> { } } - fn finalize_entry(&mut self, entry: PendingToken) -> IngestOutcome { - self.record_usage_in(entry.section); - - let sampled_token = match entry.section { - SampledTokenSection::Reasoning => SampledToken::Reasoning(entry.token), - SampledTokenSection::Content => SampledToken::Content(entry.token), - SampledTokenSection::ToolCall => SampledToken::ToolCall(entry.token), - SampledTokenSection::Pending => SampledToken::Undeterminable(entry.token), - }; - - let piece = match entry.marker_status { - PendingMarkerStatus::Unmatched => TokenPiece::Visible(entry.decoded), - PendingMarkerStatus::ResolvedBoundary | PendingMarkerStatus::AmbiguousBoundary => { - TokenPiece::Marker(entry.decoded) - } - }; - - IngestOutcome { - sampled_token, - piece, - } - } - - /// Samples a token and classifies it, appending the outcomes it finalises - /// to `outcomes`. + /// Samples a token and classifies it, appending the outcomes it settles to `outcomes`. /// /// # Errors - /// Forwards [`LlamaSampler::sample`] errors verbatim. Nothing is recorded on failure. + /// Forwards [`LlamaSampler::sample`] and [`Self::ingest`] errors verbatim. pub fn sample( &mut self, sampler: &mut LlamaSampler, @@ -415,7 +226,6 @@ impl<'model> SampledTokenClassifier<'model> { logits: bool, ) -> Result<(), BatchAddError> { batch.add(&SampledToken::Content(token), position, seq_ids, logits)?; - self.ingest_prompt_token(token); self.pending_prompt_tokens = self.pending_prompt_tokens.saturating_add(1); Ok(()) @@ -431,7 +241,6 @@ impl<'model> SampledTokenClassifier<'model> { logits_all: bool, ) -> Result<(), BatchAddError> { batch.add_sequence(tokens, seq_id, logits_all)?; - self.ingest_prompt_tokens(tokens); self.pending_prompt_tokens = self .pending_prompt_tokens .saturating_add(tokens.len() as u64); @@ -463,22 +272,25 @@ impl<'model> SampledTokenClassifier<'model> { /// them, so a chunk that cannot be decoded moves neither the KV cache nor the usage counters. /// /// # Errors - /// Returns [`EvalMultimodalChunksError::EvalFailed`] when `params.n_batch` is not positive, a - /// chunk cannot be decoded or reports a type unknown to this binding (all before any chunk is - /// evaluated), or when evaluating a chunk fails, or + /// Returns [`EvalMultimodalChunksError::EvalFailed`] when `params.n_batch` or `params.seq_id` + /// does not fit the context, a chunk cannot be decoded or reports a type unknown to this + /// binding (all before any chunk is evaluated), or when evaluating a chunk fails, or /// [`EvalMultimodalChunksError::ChunkOutOfBounds`] when a valid index returns /// `None` from `chunks.get`. pub fn eval_multimodal_chunks( &mut self, chunks: &MtmdInputChunks, mtmd_ctx: &MtmdContext, - llama_ctx: &LlamaContext, + llama_ctx: &mut LlamaContext, params: EvalMultimodalChunksParams, ) -> Result { + let limits = MtmdEvalLimits::of(llama_ctx); let fitted_chunks = FittedMtmdInputChunks::fit( chunks, mtmd_ctx, - micro_batch_tokens(llama_ctx, positive_batch_tokens(params.n_batch)?), + limits + .context + .micro_batch_tokens(limits.batch_tokens(params.n_batch, params.seq_id)?), )?; let chunk_count = fitted_chunks.len(); let mut next_position = params.start_position; @@ -506,10 +318,7 @@ impl<'model> SampledTokenClassifier<'model> { let n_tokens = fitted_chunk.chunk().n_tokens() as u64; match fitted_chunk.chunk_type() { - MtmdInputChunkType::Text => { - self.record_prompt_tokens(n_tokens); - self.ingest_prompt_tokens(fitted_chunk.text_tokens()); - } + MtmdInputChunkType::Text => self.record_prompt_tokens(n_tokens), MtmdInputChunkType::Image => self.record_input_image_tokens(n_tokens), MtmdInputChunkType::Audio => self.record_input_audio_tokens(n_tokens), } @@ -548,1184 +357,4 @@ impl<'model> SampledTokenClassifier<'model> { pub const fn current_section(&self) -> SampledTokenSection { self.section } - - #[must_use] - pub fn markers(&self) -> &StreamingMarkers { - &self.markers - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use super::PendingMarkerStatus; - use super::PendingToken; - use super::ProbeMode; - use super::SampledTokenClassifier; - use crate::bare_json_tool_calls::BareJsonToolCalls; - use crate::ingest_outcome::IngestOutcome; - use crate::marker_role::MarkerRole; - use crate::marker_role_candidate::MarkerRoleCandidate; - use crate::sampled_token::SampledToken; - use crate::sampled_token_section::SampledTokenSection; - use crate::streaming_json_probe::StreamingJsonProbe; - use crate::streaming_markers::StreamingMarkers; - use crate::token::LlamaToken; - - fn token(id: i32) -> LlamaToken { - LlamaToken::new(id) - } - - fn markers_with( - reasoning_open: Option>, - reasoning_close: Option>, - ) -> StreamingMarkers { - let candidates = reasoning_open - .into_iter() - .map(|tokens| MarkerRoleCandidate { - tokens, - role: MarkerRole::ReasoningOpen, - }) - .chain( - reasoning_close - .into_iter() - .map(|tokens| MarkerRoleCandidate { - tokens, - role: MarkerRole::ReasoningClose, - }), - ); - - StreamingMarkers::from_candidates(candidates).expect("synthetic markers must be valid") - } - - fn synthetic_classifier(markers: StreamingMarkers) -> SampledTokenClassifier<'static> { - SampledTokenClassifier { - model: unsafe { &*std::ptr::NonNull::::dangling().as_ptr() }, - marker_lookback: markers.max_token_len().saturating_sub(1), - bare_json_tool_calls: BareJsonToolCalls::Detect, - markers: Arc::new(markers), - decoder: encoding_rs::UTF_8.new_decoder(), - pending: std::collections::VecDeque::new(), - section: SampledTokenSection::Pending, - pending_prompt_tokens: 0, - usage: llama_cpp_bindings_types::TokenUsage::new(), - probe_mode: ProbeMode::Idle, - } - } - - fn push_pending(classifier: &mut SampledTokenClassifier<'_>, token_id: i32, decoded: &str) { - classifier.pending.push_back(PendingToken { - token: token(token_id), - decoded: decoded.to_owned(), - section: classifier.section, - section_before_token: classifier.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - } - - fn push_pending_from_prompt(classifier: &mut SampledTokenClassifier<'_>, token_id: i32) { - classifier.pending.push_back(PendingToken { - token: token(token_id), - decoded: String::new(), - section: classifier.section, - section_before_token: classifier.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: true, - }); - } - - fn push_and_probe( - classifier: &mut SampledTokenClassifier<'_>, - token_id: i32, - decoded: &str, - ) -> Vec { - push_pending(classifier, token_id, decoded); - classifier.try_consume_marker_at_tail(); - let mut outcomes = Vec::new(); - classifier.classify_pending_tail(&mut outcomes); - classifier.drain_overflow(&mut outcomes); - outcomes - } - - fn drained(classifier: &mut SampledTokenClassifier<'_>) -> Vec { - let mut outcomes = Vec::new(); - classifier.drain_overflow(&mut outcomes); - outcomes - } - - fn finished(classifier: &mut SampledTokenClassifier<'_>) -> Vec { - let mut outcomes = Vec::new(); - classifier.finish(&mut outcomes); - outcomes - } - - fn outcome_pieces(outcomes: &[IngestOutcome]) -> Vec<&str> { - outcomes - .iter() - .map(|outcome| outcome.piece.visible()) - .collect() - } - - fn outcome_sections(outcomes: &[IngestOutcome]) -> Vec { - outcomes - .iter() - .map(|outcome| match outcome.sampled_token { - SampledToken::Reasoning(_) => SampledTokenSection::Reasoning, - SampledToken::Content(_) => SampledTokenSection::Content, - SampledToken::ToolCall(_) => SampledTokenSection::ToolCall, - SampledToken::Undeterminable(_) => SampledTokenSection::Pending, - }) - .collect() - } - - fn markers_sharing_tool_call_open_with_a_reasoning_close( - shared: Vec, - ) -> StreamingMarkers { - StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(100)], - role: MarkerRole::ReasoningOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(200)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: shared.clone(), - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: shared, - role: MarkerRole::ToolCallOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(201)], - role: MarkerRole::ToolCallClose, - }, - ]) - .expect("synthetic markers must be valid") - } - - #[test] - fn a_token_that_is_both_a_reasoning_close_and_the_tool_call_open_opens_the_tool_call() { - let mut classifier = - synthetic_classifier(markers_sharing_tool_call_open_with_a_reasoning_close(vec![ - token(300), - ])); - classifier.section = SampledTokenSection::Content; - - push_pending(&mut classifier, 300, ""); - classifier.try_consume_marker_at_tail(); - - assert_eq!(classifier.section, SampledTokenSection::ToolCall); - } - - #[test] - fn a_shared_marker_ends_reasoning_by_opening_the_tool_call() { - let mut classifier = - synthetic_classifier(markers_sharing_tool_call_open_with_a_reasoning_close(vec![ - token(300), - ])); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 300, ""); - classifier.try_consume_marker_at_tail(); - - assert_eq!(classifier.section, SampledTokenSection::ToolCall); - } - - #[test] - fn longer_marker_reclassifies_a_completed_marker_that_was_its_prefix() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(300)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(300), token(301)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("synthetic markers must be valid"); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 300, "::new()); - - push_pending(&mut classifier, 301, "_call>"); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!(classifier.section, SampledTokenSection::ToolCall); - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::ToolCall, SampledTokenSection::ToolCall] - ); - assert_eq!(outcome_pieces(&outcomes), vec!["", ""]); - } - - #[test] - fn longer_closing_marker_uses_section_from_before_its_shorter_prefix() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(300)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(300), token(301)], - role: MarkerRole::ReasoningClose, - }, - ]); - assert!(markers.is_ok()); - let markers = markers.unwrap_or_default(); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 300, "::new()); - - push_pending(&mut classifier, 301, ">"); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!(classifier.section, SampledTokenSection::Content); - assert_eq!( - outcome_sections(&outcomes), - vec![ - SampledTokenSection::Reasoning, - SampledTokenSection::Reasoning - ] - ); - assert_eq!(outcome_pieces(&outcomes), vec!["", ""]); - assert_eq!(classifier.usage().reasoning_tokens, 2); - assert_eq!(classifier.usage().content_tokens, 0); - assert_eq!(classifier.usage().tool_call_tokens, 0); - assert_eq!(classifier.usage().undeterminable_tokens, 0); - } - - #[test] - fn single_token_close_marker_when_already_in_reasoning_emits_empty_piece_for_marker() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 7, "step"); - classifier.try_consume_marker_at_tail(); - let mut outcomes = drained(&mut classifier); - - push_pending(&mut classifier, 200, ""); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - - push_pending(&mut classifier, 9, "Hi"); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - - outcomes.extend(finished(&mut classifier)); - - assert_eq!( - outcome_sections(&outcomes), - vec![ - SampledTokenSection::Reasoning, - SampledTokenSection::Reasoning, - SampledTokenSection::Content, - ], - ); - assert_eq!(outcome_pieces(&outcomes), vec!["step", "", "Hi"]); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn multi_token_close_marker_suppresses_every_marker_token() { - let markers = markers_with( - Some(vec![token(100)]), - Some(vec![token(200), token(201), token(202)]), - ); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(7, "r"), (200, ""), (9, "OK")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!(outcome_pieces(&outcomes), vec!["r", "", "", "", "OK"]); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn marker_prefix_that_diverges_does_not_suppress_buffered_tokens() { - let markers = markers_with( - Some(vec![token(100)]), - Some(vec![token(200), token(201), token(202)]), - ); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(7, "r"), (200, "a"), (201, "b"), (300, "x")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!(outcome_pieces(&outcomes), vec!["r", "a", "b", "x"]); - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Reasoning(LlamaToken::new(0))) - })); - assert_eq!(classifier.section, SampledTokenSection::Reasoning); - } - - #[test] - fn open_then_close_back_to_back_emits_two_empty_pieces_around_zero_content() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(100, ""), (200, ""), (9, "Hi")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!( - outcome_sections(&outcomes), - vec![ - SampledTokenSection::Reasoning, - SampledTokenSection::Reasoning, - SampledTokenSection::Content, - ], - ); - assert_eq!(outcome_pieces(&outcomes), vec!["", "", "Hi"]); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn spurious_reasoning_close_in_content_section_classifies_as_content() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - push_pending(&mut classifier, 200, ""); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::Content], - ); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn spurious_tool_call_close_in_reasoning_section_classifies_as_tool_call() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(100)], - role: MarkerRole::ReasoningOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(200)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(300)], - role: MarkerRole::ToolCallOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(400)], - role: MarkerRole::ToolCallClose, - }, - ]) - .expect("synthetic markers must be valid"); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::ToolCall; - - push_pending(&mut classifier, 400, ""); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::ToolCall], - ); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn finish_releases_every_held_generated_token() { - let markers = markers_with( - Some(vec![token(100)]), - Some(vec![token(200), token(201), token(202)]), - ); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_pending(&mut classifier, 7, "abc"); - push_pending(&mut classifier, 200, "::new()); - - assert_eq!(classifier.usage().reasoning_tokens, 0); - assert_eq!(classifier.usage().content_tokens, 0); - assert_eq!(classifier.usage().tool_call_tokens, 0); - assert_eq!(classifier.usage().undeterminable_tokens, 0); - } - - #[test] - fn prompt_token_completing_marker_with_generated_token_is_suppressed_correctly() { - let markers = markers_with( - Some(vec![token(100)]), - Some(vec![token(200), token(201), token(202)]), - ); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - for token_id in [200, 201] { - push_pending_from_prompt(&mut classifier, token_id); - classifier.try_consume_marker_at_tail(); - drained(&mut classifier); - } - - assert_eq!(classifier.section, SampledTokenSection::Reasoning); - assert_eq!(classifier.pending.len(), 2); - - classifier.pending.push_back(PendingToken { - token: token(202), - decoded: "k>".to_owned(), - section: classifier.section, - section_before_token: classifier.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!(outcomes.len(), 1); - assert_eq!( - std::mem::discriminant(&outcomes[0].sampled_token), - std::mem::discriminant(&SampledToken::Reasoning(LlamaToken::new(0))) - ); - assert_eq!(outcomes[0].piece.visible(), ""); - assert_eq!(outcomes[0].piece.raw(), "k>"); - - assert_eq!(classifier.section, SampledTokenSection::Content); - assert_eq!(classifier.usage().reasoning_tokens, 1); - assert_eq!(classifier.usage().content_tokens, 0); - } - - #[test] - fn ingest_prompt_tokens_with_multiple_round_trips_ends_in_content() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - - for token_id in [100, 7, 200, 100, 8, 200] { - push_pending_from_prompt(&mut classifier, token_id); - classifier.try_consume_marker_at_tail(); - drained(&mut classifier); - } - - assert_eq!(classifier.section, SampledTokenSection::Content); - assert_eq!(classifier.usage().reasoning_tokens, 0); - assert_eq!(classifier.usage().content_tokens, 0); - assert_eq!(classifier.usage().tool_call_tokens, 0); - assert_eq!(classifier.usage().undeterminable_tokens, 0); - } - - #[test] - fn ingest_prompt_tokens_initial_section_is_always_pending() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let classifier = synthetic_classifier(markers); - - assert_eq!(classifier.section, SampledTokenSection::Pending); - } - - #[test] - fn ingest_prompt_tokens_then_drain_for_generated_token_classifies_correctly() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - - for token_id in [100, 7, 200] { - push_pending_from_prompt(&mut classifier, token_id); - classifier.try_consume_marker_at_tail(); - drained(&mut classifier); - } - - assert_eq!(classifier.section, SampledTokenSection::Content); - assert_eq!(classifier.usage().reasoning_tokens, 0); - assert_eq!(classifier.usage().content_tokens, 0); - - classifier.pending.push_back(PendingToken { - token: token(50), - decoded: "hi".to_owned(), - section: classifier.section, - section_before_token: classifier.section, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!(outcomes.len(), 1); - assert_eq!( - std::mem::discriminant(&outcomes[0].sampled_token), - std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - ); - assert_eq!(outcomes[0].piece.visible(), "hi"); - assert_eq!(classifier.usage().content_tokens, 1); - assert_eq!(classifier.usage().reasoning_tokens, 0); - assert_eq!(classifier.usage().undeterminable_tokens, 0); - } - - #[test] - fn close_marker_in_content_section_is_suppressed_as_boundary() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(7, "hi"), (200, ""), (8, "ok")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!( - outcome_sections(&outcomes), - vec![ - SampledTokenSection::Content, - SampledTokenSection::Content, - SampledTokenSection::Content, - ], - ); - assert_eq!(outcome_pieces(&outcomes), vec!["hi", "", "ok"]); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - #[test] - fn open_marker_in_reasoning_section_is_suppressed_as_boundary() { - let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - let mut outcomes = Vec::new(); - for (id, decoded) in [(7, "step1"), (100, ""), (8, "step2")] { - push_pending(&mut classifier, id, decoded); - classifier.try_consume_marker_at_tail(); - outcomes.extend(drained(&mut classifier)); - } - outcomes.extend(finished(&mut classifier)); - - assert_eq!(outcome_pieces(&outcomes), vec!["step1", "", "step2"]); - assert_eq!(classifier.section, SampledTokenSection::Reasoning); - } - - #[test] - fn record_prompt_tokens_updates_usage() { - let markers = markers_with(None, None); - let mut classifier = synthetic_classifier(markers); - - classifier.record_prompt_tokens(7); - - assert_eq!(classifier.usage().prompt_tokens, 7); - } - - #[test] - fn record_cached_prompt_tokens_updates_usage_when_under_limit() { - let markers = markers_with(None, None); - let mut classifier = synthetic_classifier(markers); - classifier.record_prompt_tokens(10); - - classifier.record_cached_prompt_tokens(3).unwrap(); - - assert_eq!(classifier.usage().cached_prompt_tokens, 3); - } - - #[test] - fn record_cached_prompt_tokens_returns_error_when_over_prompt_total() { - let markers = markers_with(None, None); - let mut classifier = synthetic_classifier(markers); - classifier.record_prompt_tokens(2); - - let result = classifier.record_cached_prompt_tokens(5); - - assert!(result.is_err()); - } - - #[test] - fn markers_accessor_returns_configured_markers() { - let configured = markers_with(Some(vec![token(1)]), Some(vec![token(2)])); - let classifier = synthetic_classifier(configured); - - let returned = classifier.markers(); - - assert!(returned.iter().any(|marker| { - marker.tokens() == [token(1)] && marker.roles() == [MarkerRole::ReasoningOpen] - })); - assert!(returned.iter().any(|marker| { - marker.tokens() == [token(2)] && marker.roles() == [MarkerRole::ReasoningClose] - })); - } - - #[test] - fn into_usage_consumes_classifier_and_yields_usage_snapshot() { - let markers = markers_with(None, None); - let mut classifier = synthetic_classifier(markers); - classifier.record_prompt_tokens(11); - - let usage = classifier.into_usage(); - - assert_eq!(usage.prompt_tokens, 11); - } - - #[test] - fn spurious_tool_call_close_in_content_section_classifies_as_content() { - let markers = StreamingMarkers::from_candidates([MarkerRoleCandidate { - tokens: vec![token(300)], - role: MarkerRole::ToolCallClose, - }]) - .expect("synthetic markers must be valid"); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - push_pending(&mut classifier, 300, ""); - classifier.try_consume_marker_at_tail(); - let outcomes = drained(&mut classifier); - - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::Content], - ); - assert_eq!(classifier.section, SampledTokenSection::Content); - } - - fn markers_with_tool_call_open(tool_call_open: Vec) -> StreamingMarkers { - StreamingMarkers::from_candidates([MarkerRoleCandidate { - tokens: tool_call_open, - role: MarkerRole::ToolCallOpen, - }]) - .expect("synthetic markers must be valid") - } - - fn feed_json_string( - classifier: &mut SampledTokenClassifier<'_>, - text: &str, - starting_token_id: i32, - ) -> Vec { - let mut outcomes = Vec::new(); - for (offset, ch) in text.char_indices() { - let token_id = starting_token_id + i32::try_from(offset).unwrap_or(i32::MAX); - let mut buffer = [0_u8; 4]; - let chunk = ch.encode_utf8(&mut buffer); - outcomes.extend(push_and_probe(classifier, token_id, chunk)); - } - outcomes - } - - #[test] - fn json_probe_engages_when_first_non_whitespace_is_open_brace_in_content() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - push_and_probe(&mut classifier, 1, "{"); - - assert_ne!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn json_probe_releases_tokens_as_tool_call_when_signature_matches() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string(&mut classifier, r#"{"name":"f","arguments":{}}"#, 100); - - assert_ne!(outcomes, Vec::::new()); - let sections = outcome_sections(&outcomes); - assert!( - sections - .iter() - .all(|section| *section == SampledTokenSection::ToolCall), - "every emitted outcome should be ToolCall, got {sections:?}", - ); - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn json_probe_releases_tokens_as_content_when_signature_does_not_match() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string(&mut classifier, r#"{"foo":"bar"}"#, 100); - - let sections = outcome_sections(&outcomes); - assert!( - sections - .iter() - .all(|section| *section == SampledTokenSection::Content), - "every emitted outcome should be Content, got {sections:?}", - ); - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn json_probe_releases_tokens_as_content_when_extra_top_level_key() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{},"extra":1}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_releases_tokens_as_content_when_arguments_is_not_object() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string(&mut classifier, r#"{"name":"f","arguments":"hi"}"#, 100); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_strings_with_quoted_braces_in_arguments() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{"q":"a } b"}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_escaped_quotes_in_string_values() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{"q":"he said \"hi\""}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_unicode_letters_in_strings() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"日本語","arguments":{"city":"パリ"}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_nested_objects() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{"a":{"b":{"c":1}}}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_handles_arrays_inside_arguments() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - r#"{"name":"f","arguments":{"items":[1,2,3]}}"#, - 100, - ); - - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_does_not_engage_when_first_byte_is_close_brace() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string(&mut classifier, "}}", 100); - - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - assert!(outcomes.iter().all(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - })); - } - - #[test] - fn json_probe_does_not_engage_in_reasoning_section() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(800)], - role: MarkerRole::ReasoningOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(801)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(900)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("synthetic markers must be valid"); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Reasoning; - - push_and_probe(&mut classifier, 1, "{"); - - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn json_probe_does_not_engage_when_bare_json_tool_calls_are_ignored() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.bare_json_tool_calls = BareJsonToolCalls::Ignore; - classifier.section = SampledTokenSection::Content; - - let outcomes = push_and_probe(&mut classifier, 1, "{"); - - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - assert_eq!(outcome_pieces(&outcomes), vec!["{"]); - } - - #[test] - fn json_probe_does_not_engage_in_tool_call_section() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::ToolCall; - - push_and_probe(&mut classifier, 1, "{"); - - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn marker_probe_takes_precedence_when_both_could_match() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let mut outcomes = Vec::new(); - outcomes.extend(push_and_probe(&mut classifier, 1, "{")); - outcomes.extend(push_and_probe(&mut classifier, 900, r#"""#)); - - assert_eq!(classifier.section, SampledTokenSection::ToolCall); - assert_eq!(outcome_pieces(&outcomes), vec!["{", ""]); - assert_eq!( - outcome_sections(&outcomes), - vec![SampledTokenSection::Content, SampledTokenSection::ToolCall], - ); - } - - #[test] - fn json_probe_consumes_two_consecutive_objects_separately() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let mut outcomes = Vec::new(); - outcomes.extend(feed_json_string( - &mut classifier, - r#"{"name":"a","arguments":{}}"#, - 100, - )); - outcomes.extend(feed_json_string( - &mut classifier, - r#"{"name":"b","arguments":{"x":1}}"#, - 200, - )); - - let sections = outcome_sections(&outcomes); - assert!( - sections - .iter() - .all(|section| *section == SampledTokenSection::ToolCall), - "two consecutive markerless tool calls must both classify as ToolCall, got {sections:?}", - ); - } - - #[test] - fn json_probe_with_leading_whitespace_then_open_brace_classifies_whitespace_as_content_and_json_as_tool_call() - { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let outcomes = feed_json_string( - &mut classifier, - "\n {\"name\":\"f\",\"arguments\":{}}", - 100, - ); - - let tool_call_count = outcomes - .iter() - .filter(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::ToolCall(LlamaToken::new(0))) - }) - .count(); - let content_count = outcomes - .iter() - .filter(|outcome| { - std::mem::discriminant(&outcome.sampled_token) - == std::mem::discriminant(&SampledToken::Content(LlamaToken::new(0))) - }) - .count(); - assert_eq!( - content_count, 3, - "leading `\\n ` should classify as content" - ); - assert!( - tool_call_count > 0, - "the JSON object should classify as ToolCall", - ); - assert_eq!(content_count + tool_call_count, outcomes.len()); - } - - #[test] - fn json_probe_records_tool_call_token_usage_on_commit() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let json = r#"{"name":"f","arguments":{}}"#; - let outcomes = feed_json_string(&mut classifier, json, 100); - - let emitted = outcomes.len(); - let usage = classifier.usage(); - assert_eq!(usage.tool_call_tokens, emitted as u64); - assert_eq!(usage.content_tokens, 0); - } - - #[test] - fn json_probe_records_content_token_usage_on_abandon() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - let json = r#"{"foo":"bar"}"#; - let outcomes = feed_json_string(&mut classifier, json, 100); - - let emitted = outcomes.len(); - let usage = classifier.usage(); - assert_eq!(usage.content_tokens, emitted as u64); - assert_eq!(usage.tool_call_tokens, 0); - } - - #[test] - fn finish_during_active_json_probe_releases_held_tokens_as_content() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - push_and_probe(&mut classifier, 1, "{"); - push_and_probe(&mut classifier, 2, r#""name""#); - assert_ne!(classifier.probe_mode, ProbeMode::Idle); - - let outcomes = finished(&mut classifier); - - let sections = outcome_sections(&outcomes); - assert!( - sections - .iter() - .all(|section| *section == SampledTokenSection::Content), - "finishing mid-probe must release held tokens as Content, got {sections:?}", - ); - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } - - #[test] - fn commit_probe_as_tool_call_requeues_non_held_entries_and_releases_held_as_tool_call() { - let markers = markers_with_tool_call_open(vec![token(900)]); - let mut classifier = synthetic_classifier(markers); - classifier.section = SampledTokenSection::Content; - - classifier.pending.push_back(PendingToken { - token: token(1), - decoded: "before".to_owned(), - section: SampledTokenSection::Content, - section_before_token: SampledTokenSection::Content, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - classifier.pending.push_back(PendingToken { - token: token(2), - decoded: "{}".to_owned(), - section: SampledTokenSection::Content, - section_before_token: SampledTokenSection::Content, - marker_status: PendingMarkerStatus::Unmatched, - is_from_prompt: false, - }); - classifier.probe_mode = ProbeMode::Active { - probe: StreamingJsonProbe::default(), - held_count: 1, - }; - - let mut outcomes = Vec::new(); - classifier.commit_probe_as_tool_call(&mut outcomes); - - let sections = outcome_sections(&outcomes); - assert_eq!(sections, vec![SampledTokenSection::ToolCall]); - assert_eq!(classifier.pending.len(), 1); - assert_eq!(classifier.pending[0].token, token(1)); - assert_eq!(classifier.probe_mode, ProbeMode::Idle); - } } diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index a950cc6f6..03d344d8c 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -1,21 +1,32 @@ use std::borrow::Borrow; use std::ffi::{CString, c_char}; +use std::fmt; use std::fmt::{Debug, Formatter}; +use std::mem; +use std::ptr; use std::ptr::NonNull; -use llama_cpp_error_recorder::ErrorScope; -use llama_cpp_error_recorder::RecordedError; +use llama_cpp_error_recorder::error_scope::ErrorScope; +use llama_cpp_error_recorder::recorded_error::RecordedError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; +use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; +use llama_cpp_gbnf::validate_gbnf::validate_gbnf; use crate::context::LlamaContext; +use crate::dry_sequence_breakers::dry_sequence_breakers; +use crate::error::grammar_error::GrammarError; +use crate::error::sample_error::SampleError; +use crate::error::sampler_accept_error::SamplerAcceptError; +use crate::error::sampler_apply_error::SamplerApplyError; +use crate::error::sampling_error::SamplingError; +use crate::llguidance_sampler::create_llg_sampler; use crate::model::LlamaModel; use crate::sanitized_grammar::SanitizedGrammar; use crate::token::LlamaToken; use crate::token::data_array::LlamaTokenDataArray; use crate::token::logit_bias::LlamaLogitBias; -use crate::{GrammarError, SampleError, SamplerAcceptError, SamplingError}; -use llama_cpp_ffi_status::read_and_free_cpp_string; -use llama_cpp_gbnf::gbnf_validation_error::GbnfValidationError; -use llama_cpp_gbnf::validate_gbnf::validate_gbnf; fn check_sampler_accept_status( status: llama_cpp_bindings_sys::llama_rs_sampler_accept_status, @@ -39,21 +50,19 @@ fn check_sampler_accept_status( }?; Err(SamplerAcceptError::GrammarStateCorrupted { message }) } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_SAMPLER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_sampler_accept", - detail: "was given a null sampler argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_SAMPLER_ARG => Err(FfiContractError { + operation: "llama_rs_sampler_accept", + detail: "was given a null sampler argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_accept", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_accept", code: i64::from(other), } @@ -61,6 +70,53 @@ fn check_sampler_accept_status( } } +fn sampler_apply_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_sampler_apply_status, + out_error: *mut c_char, +) -> Result<(), SamplerApplyError> { + match status { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_OK => Ok(()), + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_SAMPLER_ARG => { + Err(SamplerApplyError::NullSampler) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED => { + Err(SamplerApplyError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_OUT_OF_MEMORY => { + Err(SamplerApplyError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_sampler_apply", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(SamplerApplyError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG => { + Err(FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "was given a null data_array argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG => { + Err(FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "was given a null out_error argument", + } + .into()) + } + other => Err(FfiStatusError { + operation: "llama_rs_sampler_apply", + code: i64::from(other), + } + .into()), + } +} + fn sampler_sample_status_to_result( status: llama_cpp_bindings_sys::llama_rs_sampler_sample_status, token: i32, @@ -84,35 +140,31 @@ fn sampler_sample_status_to_result( }?; Err(SampleError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_SAMPLER_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_sampler_sample", - detail: "was given a null sampler argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_SAMPLER_ARG => Err(FfiContractError { + operation: "llama_rs_sampler_sample", + detail: "was given a null sampler argument", } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_sampler_sample", - detail: "was given a null ctx argument", - } - .into()) + .into()), + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_CTX_ARG => Err(FfiContractError { + operation: "llama_rs_sampler_sample", + detail: "was given a null ctx argument", } + .into()), llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_TOKEN_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_token argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_sample", code: i64::from(other), } @@ -127,7 +179,7 @@ fn sampler_init_grammar_status_to_result( ) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_OK => { - LlamaSampler::from_raw(sampler, "grammar").map_err(Into::into) + unsafe { LlamaSampler::from_raw(sampler, "grammar") }.map_err(Into::into) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_RETURNED_NULL => { Err(GrammarError::GrammarMalformed) @@ -149,20 +201,20 @@ fn sampler_init_grammar_status_to_result( Err(GrammarError::Reported { message }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_SAMPLER_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_sampler argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_init_grammar", code: i64::from(other), } @@ -177,7 +229,7 @@ fn sampler_init_grammar_lazy_patterns_status_to_result( ) -> Result { match status { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_OK => { - LlamaSampler::from_raw(sampler, "lazy grammar").map_err(Into::into) + unsafe { LlamaSampler::from_raw(sampler, "lazy grammar") }.map_err(Into::into) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_RETURNED_NULL => { Err(GrammarError::LazyGrammarMalformed) @@ -196,17 +248,17 @@ fn sampler_init_grammar_lazy_patterns_status_to_result( let message = unsafe { read_and_free_cpp_string(error_ptr, "llama_rs_sampler_init_grammar_lazy_patterns", "reported a thrown C++ exception without an error message") }?; Err(GrammarError::Reported { message }) } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_SAMPLER_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_SAMPLER_ARG => Err(FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_sampler argument", } .into()), - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG => Err(crate::FfiContractError { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG => Err(FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_error argument", } .into()), - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", code: i64::from(other), } @@ -221,9 +273,9 @@ fn sampler_init_status_to_result( sampler_name: &'static str, ) -> Result { match status { - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_OK => { + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_OK => unsafe { LlamaSampler::from_raw(sampler, sampler_name) - } + }, llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_LLAMA_CPP_RETURNED_NULL => { Err(SamplingError::SamplerUnavailable { sampler: sampler_name, @@ -251,20 +303,18 @@ fn sampler_init_status_to_result( }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_SAMPLER_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_sampler argument", } .into()) } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_sampler_init", - detail: "was given a null out_error argument", - } - .into()) + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_ERROR_ARG => Err(FfiContractError { + operation: "llama_rs_sampler_init", + detail: "was given a null out_error argument", } - other => Err(crate::FfiStatusError { + .into()), + other => Err(FfiStatusError { operation: "llama_rs_sampler_init", code: i64::from(other), } @@ -298,27 +348,27 @@ fn sampler_chain_add_status_to_result( }) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_CHAIN_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null chain argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_SAMPLER_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null sampler argument", } .into()) } llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null out_error argument", } .into()) } - other => Err(crate::FfiStatusError { + other => Err(FfiStatusError { operation: "llama_rs_sampler_chain_add", code: i64::from(other), } @@ -355,13 +405,23 @@ fn grammar_callback_error_to_accept_result( } impl Debug for LlamaSampler { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LlamaSamplerChain").finish() + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("LlamaSampler").finish_non_exhaustive() } } impl LlamaSampler { - pub(crate) fn from_raw( + /// Takes ownership of a sampler llama.cpp initialized, and frees it on drop. + /// + /// # Safety + /// + /// `sampler` must be null or a sampler llama.cpp initialized that no other + /// [`LlamaSampler`] owns. + /// + /// # Errors + /// + /// Returns [`SamplingError::SamplerUnavailable`] when `sampler` is null. + pub unsafe fn from_raw( sampler: *mut llama_cpp_bindings_sys::llama_sampler, sampler_name: &'static str, ) -> Result { @@ -385,15 +445,15 @@ impl LlamaSampler { *mut *mut c_char, ) -> llama_cpp_bindings_sys::llama_rs_sampler_init_status, ) -> Result { - let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = std::ptr::null_mut(); - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let status = initialize_sampler(&raw mut sampler, &raw mut error_ptr); sampler_init_status_to_result(status, sampler, error_ptr, sampler_name) } fn take_into_chain(&self, sampler: Self) -> Result<(), SamplingError> { - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_sampler_chain_add( self.sampler.as_ptr(), @@ -402,7 +462,7 @@ impl LlamaSampler { ) }; - sampler_chain_add_status_to_result(status, error_ptr).map(|()| std::mem::forget(sampler)) + sampler_chain_add_status_to_result(status, error_ptr).map(|()| mem::forget(sampler)) } #[must_use] @@ -412,11 +472,14 @@ impl LlamaSampler { /// # Errors /// - /// Returns [`SampleError`] if the C++ sampler throws an exception, the index is invalid, or the - /// grammar sampler callback recorded a failure during sampling. + /// Returns [`SampleError::Logits`] when the context holds no logits for `idx`, or another + /// [`SampleError`] if the C++ sampler throws an exception or the grammar sampler callback + /// recorded a failure during sampling. pub fn sample(&mut self, ctx: &LlamaContext, idx: i32) -> Result { + ctx.get_logits_ith(idx)?; + let mut token: i32 = -1; - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let scope = ErrorScope::enter(); let status = unsafe { @@ -442,16 +505,40 @@ impl LlamaSampler { /// every candidate. pub fn apply(&self, data_array: &mut LlamaTokenDataArray) -> Result<(), SampleError> { let scope = ErrorScope::enter(); - let applied = data_array.apply_sampler(self); + let applied = self.apply_to_candidates(data_array); grammar_callback_error_to_result(scope.take())?; Ok(applied?) } + fn apply_to_candidates( + &self, + data_array: &mut LlamaTokenDataArray, + ) -> Result<(), SamplerApplyError> { + unsafe { + data_array.modify_as_c_llama_token_data_array(|c_llama_token_data_array| { + let mut out_error: *mut c_char = ptr::null_mut(); + let status = llama_cpp_bindings_sys::llama_rs_sampler_apply( + self.sampler.as_ptr(), + c_llama_token_data_array, + &raw mut out_error, + ); + + sampler_apply_status_to_result(status, out_error) + }) + }??; + + if data_array.has_selectable_candidate() { + Ok(()) + } else { + Err(SamplerApplyError::EveryCandidateExcluded) + } + } + /// # Errors /// Returns [`SamplerAcceptError`] if the underlying sampler rejects the token. pub fn accept(&mut self, token: LlamaToken) -> Result<(), SamplerAcceptError> { - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let scope = ErrorScope::enter(); let status = unsafe { @@ -480,17 +567,6 @@ impl LlamaSampler { Ok(()) } - /// # Errors - /// Returns [`SamplerAcceptError`] if the underlying sampler rejects any token. - pub fn with_tokens( - mut self, - tokens: impl IntoIterator>, - ) -> Result { - self.accept_many(tokens)?; - - Ok(self) - } - /// # Errors /// /// Returns [`SampleError`] if the grammar sampler callback recorded a failure during reset. @@ -511,13 +587,10 @@ impl LlamaSampler { /// # Errors /// /// Returns [`SamplingError`] if the sampler chain cannot be initialized. - pub fn chain( - samplers: impl IntoIterator, - no_perf: bool, - ) -> Result { + pub fn chain(samplers: impl IntoIterator) -> Result { Self::initialize("chain", |out_sampler, out_error| unsafe { llama_cpp_bindings_sys::llama_rs_sampler_chain_init( - llama_cpp_bindings_sys::llama_sampler_chain_params { no_perf }, + llama_cpp_bindings_sys::llama_sampler_chain_params { no_perf: true }, out_sampler, out_error, ) @@ -530,13 +603,6 @@ impl LlamaSampler { }) } - /// # Errors - /// - /// Returns [`SamplingError`] if the sampler chain cannot be initialized. - pub fn chain_simple(samplers: impl IntoIterator) -> Result { - Self::chain(samplers, false) - } - /// # Errors /// /// Returns [`SamplingError`] if the temperature sampler cannot be initialized. @@ -657,8 +723,8 @@ impl LlamaSampler { grammar: grammar_cstring, root: root_cstring, } = Self::sanitize_grammar_strings(grammar_str, grammar_root)?; - let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = std::ptr::null_mut(); - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let status = unsafe { llama_cpp_bindings_sys::llama_rs_sampler_init_grammar( @@ -689,8 +755,8 @@ impl LlamaSampler { root: root_cstring, } = Self::sanitize_grammar_strings(grammar_str, grammar_root)?; let trigger_patterns = Self::sanitize_trigger_patterns(trigger_patterns)?; - let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = std::ptr::null_mut(); - let mut error_ptr: *mut c_char = std::ptr::null_mut(); + let mut sampler: *mut llama_cpp_bindings_sys::llama_sampler = ptr::null_mut(); + let mut error_ptr: *mut c_char = ptr::null_mut(); let mut trigger_pattern_ptrs: Vec<*const c_char> = trigger_patterns.iter().map(|cs| cs.as_ptr()).collect(); @@ -724,7 +790,7 @@ impl LlamaSampler { grammar_kind: &str, grammar_data: &str, ) -> Result { - crate::llguidance_sampler::create_llg_sampler(model, grammar_kind, grammar_data) + create_llg_sampler(model, grammar_kind, grammar_data) } fn sanitize_grammar_strings( @@ -771,7 +837,8 @@ impl LlamaSampler { } /// # Errors - /// Returns an error if any string in `seq_breakers` contains null bytes. + /// Returns the [`SamplingError`] describing a sequence breaker llama.cpp would skip, truncate, + /// or ignore, or the error of initializing the sampler. pub fn dry( model: &LlamaModel, multiplier: f32, @@ -779,13 +846,8 @@ impl LlamaSampler { allowed_length: i32, penalty_last_n: i32, seq_breakers: impl IntoIterator>, - ) -> Result { - let seq_breakers: Vec = seq_breakers - .into_iter() - .map(|seq_breaker| { - CString::new(seq_breaker.as_ref()).map_err(GrammarError::SequenceBreakerContainsNul) - }) - .collect::, _>>()?; + ) -> Result { + let seq_breakers = dry_sequence_breakers(seq_breakers)?; let mut seq_breaker_pointers: Vec<*const c_char> = seq_breakers .iter() .map(|seq_breaker| seq_breaker.as_ptr()) @@ -804,7 +866,6 @@ impl LlamaSampler { out_error, ) }) - .map_err(Into::into) } /// # Errors @@ -917,16 +978,23 @@ impl Drop for LlamaSampler { #[cfg(test)] mod tests { - use std::ffi::CString; + use std::ffi::{CStr, CString}; use std::ptr; + use std::slice; - use llama_cpp_error_recorder::RecordedError; - use llama_cpp_error_recorder::record; + use llama_cpp_error_recorder::record::record; + use llama_cpp_error_recorder::recorded_error::RecordedError; use super::LlamaSampler; - use crate::GrammarError; - use crate::SampleError; + use crate::error::grammar_error::GrammarError; + use crate::error::sample_error::SampleError; + use crate::error::sampler_apply_error::SamplerApplyError; + use crate::error::sampling_error::SamplingError; use crate::sanitized_grammar::SanitizedGrammar; + use crate::token::LlamaToken; + use crate::token::data::LlamaTokenData; + use crate::token::data_array::LlamaTokenDataArray; + use crate::token::logit_bias::LlamaLogitBias; unsafe extern "C" fn record_a_failure_and_exclude_every_candidate( _sampler: *mut llama_cpp_bindings_sys::llama_sampler, @@ -935,8 +1003,7 @@ mod tests { record(RecordedError::new("mask failed".to_string())); let candidates = unsafe { &mut *candidates }; - let candidate_data = - unsafe { std::slice::from_raw_parts_mut(candidates.data, candidates.size) }; + let candidate_data = unsafe { slice::from_raw_parts_mut(candidates.data, candidates.size) }; for candidate in candidate_data { candidate.logit = f32::NEG_INFINITY; } @@ -975,16 +1042,12 @@ mod tests { ) }; - LlamaSampler::from_raw(sampler, "failing excluding sampler") + unsafe { LlamaSampler::from_raw(sampler, "failing excluding sampler") } .expect("the failing excluding sampler must initialize") } #[test] fn a_grammar_callback_failure_takes_precedence_over_a_failed_application() { - use crate::token::LlamaToken; - use crate::token::data::LlamaTokenData; - use crate::token::data_array::LlamaTokenDataArray; - let mut data_array = LlamaTokenDataArray::new( vec![ LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), @@ -1002,17 +1065,17 @@ mod tests { } #[test] - fn null_native_sampler_is_initialization_error() { - let result = LlamaSampler::from_raw(ptr::null_mut(), "test"); + fn from_raw_reports_a_null_native_sampler_as_unavailable() { + let result = unsafe { LlamaSampler::from_raw(ptr::null_mut(), "test") }; assert_eq!( result.unwrap_err(), - crate::SamplingError::SamplerUnavailable { sampler: "test" } + SamplingError::SamplerUnavailable { sampler: "test" } ); } #[test] - fn sanitize_grammar_strings_valid() { + fn sanitize_grammar_strings_converts_a_grammar_and_root_without_nul_bytes() { assert_eq!( LlamaSampler::sanitize_grammar_strings("root ::= \"hello\"", "root"), Ok(SanitizedGrammar { @@ -1023,7 +1086,7 @@ mod tests { } #[test] - fn sanitize_grammar_strings_null_byte_in_root_only() { + fn sanitize_grammar_strings_reports_a_nul_byte_in_the_root() { assert_eq!( LlamaSampler::sanitize_grammar_strings("root ::= \"hello\"", "ro\0ot"), Err(GrammarError::RootContainsNul( @@ -1033,7 +1096,7 @@ mod tests { } #[test] - fn sanitize_grammar_strings_null_byte_in_grammar() { + fn sanitize_grammar_strings_reports_a_nul_byte_in_the_grammar() { assert_eq!( LlamaSampler::sanitize_grammar_strings("root ::= \"\0\"", "root"), Err(GrammarError::GrammarContainsNul( @@ -1043,7 +1106,7 @@ mod tests { } #[test] - fn sanitize_grammar_strings_null_byte_in_root() { + fn sanitize_grammar_strings_reports_a_nul_byte_in_the_grammar_before_one_in_the_root() { assert_eq!( LlamaSampler::sanitize_grammar_strings("ro\0ot ::= \"hello\"", "ro\0ot"), Err(GrammarError::GrammarContainsNul( @@ -1054,28 +1117,27 @@ mod tests { } #[test] - fn sanitize_trigger_patterns_valid() { + fn sanitize_trigger_patterns_converts_every_pattern() { let patterns = vec!["^hello$".to_string(), "world.*".to_string()]; - let result = LlamaSampler::sanitize_trigger_patterns(&patterns); - assert!(result.is_ok()); - assert_eq!(result.expect("valid trigger patterns").len(), 2); + assert_eq!( + LlamaSampler::sanitize_trigger_patterns(&patterns), + Ok(vec![c"^hello$".to_owned(), c"world.*".to_owned()]) + ); } #[test] - fn sanitize_trigger_patterns_empty_list() { + fn sanitize_trigger_patterns_converts_an_empty_list_to_no_patterns() { let patterns: Vec = vec![]; - let result = LlamaSampler::sanitize_trigger_patterns(&patterns); - assert!(result.is_ok()); assert_eq!( - result.expect("valid trigger patterns"), - Vec::::new() + LlamaSampler::sanitize_trigger_patterns(&patterns), + Ok(Vec::::new()) ); } #[test] - fn sanitize_trigger_patterns_null_byte() { + fn sanitize_trigger_patterns_reports_a_nul_byte_in_a_pattern() { let patterns = vec!["hel\0lo".to_string()]; assert_eq!( LlamaSampler::sanitize_trigger_patterns(&patterns), @@ -1086,11 +1148,7 @@ mod tests { } #[test] - fn apply_modifies_data_array() { - use crate::token::LlamaToken; - use crate::token::data::LlamaTokenData; - use crate::token::data_array::LlamaTokenDataArray; - + fn a_greedy_sampler_selects_the_highest_logit() { let sampler = greedy_sampler(); let mut data_array = LlamaTokenDataArray::new( vec![ @@ -1100,19 +1158,12 @@ mod tests { false, ); - assert!(sampler.apply(&mut data_array).is_ok()); - + assert_eq!(sampler.apply(&mut data_array), Ok(())); assert_eq!(data_array.selected_token(), Some(LlamaToken::new(1))); } #[test] fn apply_surfaces_a_sampler_that_excludes_every_candidate() { - use crate::error::sampler_apply_error::SamplerApplyError; - use crate::token::LlamaToken; - use crate::token::data::LlamaTokenData; - use crate::token::data_array::LlamaTokenDataArray; - use crate::token::logit_bias::LlamaLogitBias; - let excluding_sampler = LlamaSampler::logit_bias( 2, &[ @@ -1138,108 +1189,119 @@ mod tests { } #[test] - fn accept_succeeds() { - let mut sampler = LlamaSampler::chain_simple([penalties_sampler(), greedy_sampler()]) + fn a_penalties_chain_accepts_a_token() { + let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); - sampler - .accept(crate::token::LlamaToken::new(1)) - .expect("test: accept should succeed"); + assert_eq!(sampler.accept(LlamaToken::new(1)), Ok(())); } #[test] - fn accept_succeeds_on_penalties_sampler() { - let mut sampler = LlamaSampler::chain_simple([penalties_sampler(), greedy_sampler()]) + fn a_penalties_chain_accepts_many_tokens() { + let mut sampler = LlamaSampler::chain([penalties_sampler(), greedy_sampler()]) .expect("sampler chain must initialize"); - let result = sampler.accept(crate::token::LlamaToken::new(42)); - - assert!(result.is_ok()); + assert_eq!( + sampler.accept_many([LlamaToken::new(1), LlamaToken::new(2), LlamaToken::new(3)]), + Ok(()) + ); } #[test] - fn accept_many_multiple_tokens() { - use crate::token::LlamaToken; - - let mut sampler = LlamaSampler::chain_simple([penalties_sampler(), greedy_sampler()]) - .expect("sampler chain must initialize"); - - sampler - .accept_many([LlamaToken::new(1), LlamaToken::new(2), LlamaToken::new(3)]) - .expect("test: accept_many should succeed"); - } + fn every_model_free_sampler_constructor_builds_the_matching_llama_cpp_sampler() { + let biases = vec![LlamaLogitBias::new(LlamaToken::new(0), -100.0)]; + let samplers = [ + LlamaSampler::temp(0.8), + LlamaSampler::temp_ext(0.8, 0.1, 1.0), + LlamaSampler::top_k(40), + LlamaSampler::top_n_sigma(2.0), + LlamaSampler::top_p(0.9, 1), + LlamaSampler::min_p(0.05, 1), + LlamaSampler::typical(0.9, 1), + LlamaSampler::xtc(0.1, 0.5, 1, 42), + LlamaSampler::dist(42), + LlamaSampler::mirostat(32000, 42, 5.0, 0.1, 100), + LlamaSampler::mirostat_v2(42, 5.0, 0.1), + LlamaSampler::penalties(32_000, 64, 1.1, 0.0, 0.0), + LlamaSampler::logit_bias(32000, &biases), + LlamaSampler::greedy(), + LlamaSampler::chain([greedy_sampler()]), + ]; - #[test] - fn with_tokens_builder_pattern() { - use crate::token::LlamaToken; + assert_eq!( + samplers.map(|initialized_sampler| { + initialized_sampler.map(|sampler| { + unsafe { + CStr::from_ptr(llama_cpp_bindings_sys::llama_sampler_name(sampler.as_ptr())) + } + .to_owned() + }) + }), + [ + Ok(c"temp".to_owned()), + Ok(c"temp-ext".to_owned()), + Ok(c"top-k".to_owned()), + Ok(c"top-n-sigma".to_owned()), + Ok(c"top-p".to_owned()), + Ok(c"min-p".to_owned()), + Ok(c"typical".to_owned()), + Ok(c"xtc".to_owned()), + Ok(c"dist".to_owned()), + Ok(c"mirostat".to_owned()), + Ok(c"mirostat-v2".to_owned()), + Ok(c"penalties".to_owned()), + Ok(c"logit-bias".to_owned()), + Ok(c"greedy".to_owned()), + Ok(c"chain".to_owned()), + ] + ); + } + + #[test] + fn resetting_a_sampler_succeeds_without_a_grammar_callback_failure() { + let mut sampler = LlamaSampler::dist(42).expect("distribution sampler must initialize"); - let _sampler = LlamaSampler::chain_simple([penalties_sampler(), greedy_sampler()]) - .expect("sampler chain must initialize") - .with_tokens([LlamaToken::new(10), LlamaToken::new(20)]) - .expect("test: with_tokens should succeed"); + assert_eq!(sampler.reset(), Ok(())); } #[test] - fn all_sampler_constructors() { - use crate::token::LlamaToken; - use crate::token::logit_bias::LlamaLogitBias; - - let _temp = LlamaSampler::temp(0.8).expect("temperature sampler must initialize"); - let _temp_ext = LlamaSampler::temp_ext(0.8, 0.1, 1.0) - .expect("extended temperature sampler must initialize"); - let _top_k = LlamaSampler::top_k(40).expect("top-k sampler must initialize"); - let _top_n_sigma = - LlamaSampler::top_n_sigma(2.0).expect("top-n-sigma sampler must initialize"); - let _top_p = LlamaSampler::top_p(0.9, 1).expect("top-p sampler must initialize"); - let _min_p = LlamaSampler::min_p(0.05, 1).expect("min-p sampler must initialize"); - let _typical = LlamaSampler::typical(0.9, 1).expect("typical sampler must initialize"); - let _xtc = LlamaSampler::xtc(0.1, 0.5, 1, 42).expect("XTC sampler must initialize"); - let _dist = LlamaSampler::dist(42).expect("distribution sampler must initialize"); - let _mirostat = LlamaSampler::mirostat(32000, 42, 5.0, 0.1, 100) - .expect("Mirostat sampler must initialize"); - let _mirostat_v2 = - LlamaSampler::mirostat_v2(42, 5.0, 0.1).expect("Mirostat v2 sampler must initialize"); - let biases = vec![LlamaLogitBias::new(LlamaToken::new(0), -100.0)]; - let _logit_bias = LlamaSampler::logit_bias(32000, &biases); - let _chain = - LlamaSampler::chain([greedy_sampler()], true).expect("sampler chain must initialize"); - } + fn a_distribution_sampler_reports_the_seed_it_was_built_with() { + let sampler = LlamaSampler::dist(42).expect("distribution sampler must initialize"); - #[test] - fn reset_and_get_seed() { - let mut sampler = LlamaSampler::dist(42).expect("distribution sampler must initialize"); - assert!(sampler.reset().is_ok()); - let _seed = sampler.get_seed(); + assert_eq!(sampler.get_seed(), 42); } #[test] - fn debug_formatting() { + fn debug_hides_the_native_sampler_pointer() { let sampler = greedy_sampler(); let debug_output = format!("{sampler:?}"); - assert!(debug_output.contains("LlamaSampler")); + assert_eq!(debug_output, "LlamaSampler { .. }"); } -} - -#[cfg(test)] -mod checked_usize_as_i32_sampling_tests { - use super::checked_usize_as_i32_sampling; #[test] - fn checked_usize_as_i32_sampling_overflow() { - let result = checked_usize_as_i32_sampling(usize::MAX); - assert!(result.is_err()); + fn a_logit_bias_list_longer_than_a_c_int_is_refused() { + let bias_count = usize::try_from(i32::MAX).unwrap() + 1; + let zeroed_biases = + unsafe { Box::<[LlamaLogitBias]>::new_zeroed_slice(bias_count).assume_init() }; + + assert_eq!( + LlamaSampler::logit_bias(1, &zeroed_biases).err(), + i32::try_from(bias_count) + .err() + .map(SamplingError::IntegerOverflow) + ); } } #[cfg(test)] mod grammar_callback_error_to_result_tests { - use llama_cpp_error_recorder::RecordedError; + use llama_cpp_error_recorder::recorded_error::RecordedError; use super::grammar_callback_error_to_result; - use crate::SampleError; + use crate::error::sample_error::SampleError; #[test] - fn grammar_callback_error_to_result_maps_recorded_error() { + fn grammar_callback_error_to_result_reports_the_recorded_grammar_callback_failure() { let result = grammar_callback_error_to_result(Some(RecordedError::new("mask failed".to_string()))); @@ -1252,20 +1314,20 @@ mod grammar_callback_error_to_result_tests { } #[test] - fn grammar_callback_error_to_result_maps_absence_to_ok() { - assert!(grammar_callback_error_to_result(None).is_ok()); + fn grammar_callback_error_to_result_succeeds_without_a_recorded_error() { + assert_eq!(grammar_callback_error_to_result(None), Ok(())); } } #[cfg(test)] mod grammar_callback_error_to_accept_result_tests { - use llama_cpp_error_recorder::RecordedError; + use llama_cpp_error_recorder::recorded_error::RecordedError; use super::grammar_callback_error_to_accept_result; - use crate::SamplerAcceptError; + use crate::error::sampler_accept_error::SamplerAcceptError; #[test] - fn grammar_callback_error_to_accept_result_maps_recorded_error() { + fn grammar_callback_error_to_accept_result_reports_the_recorded_grammar_callback_failure() { let result = grammar_callback_error_to_accept_result(Some(RecordedError::new( "consume failed".to_string(), ))); @@ -1279,30 +1341,35 @@ mod grammar_callback_error_to_accept_result_tests { } #[test] - fn grammar_callback_error_to_accept_result_maps_absence_to_ok() { - assert!(grammar_callback_error_to_accept_result(None).is_ok()); + fn grammar_callback_error_to_accept_result_succeeds_without_a_recorded_error() { + assert_eq!(grammar_callback_error_to_accept_result(None), Ok(())); } } #[cfg(test)] mod check_sampler_accept_status_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::check_sampler_accept_status; - use crate::SamplerAcceptError; + use crate::error::sampler_accept_error::SamplerAcceptError; #[test] - fn check_sampler_accept_status_ok() { - let result = check_sampler_accept_status( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_OK, - ptr::null_mut(), + fn check_sampler_accept_status_maps_ok_to_success() { + assert_eq!( + check_sampler_accept_status( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_OK, + ptr::null_mut(), + ), + Ok(()) ); - - assert!(result.is_ok()); } #[test] - fn check_sampler_accept_status_exception_maps_to_typed_variant() { + fn check_sampler_accept_status_reports_an_exception_as_a_corrupted_grammar_state() { let out_error = unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"grammar state corrupted".as_ptr()) }; @@ -1335,7 +1402,7 @@ mod check_sampler_accept_status_tests { assert_eq!( result, - Err(SamplerAcceptError::FfiContract(crate::FfiContractError { + Err(SamplerAcceptError::FfiContract(FfiContractError { operation: "llama_rs_sampler_accept", detail: "was given a null sampler argument", })) @@ -1343,7 +1410,7 @@ mod check_sampler_accept_status_tests { } #[test] - fn check_sampler_accept_status_maps_null_out_error_arg() { + fn check_sampler_accept_status_reports_a_null_out_error_arg_as_a_contract_violation() { let outcome = check_sampler_accept_status( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_NULL_OUT_ERROR_ARG, ptr::null_mut(), @@ -1351,7 +1418,7 @@ mod check_sampler_accept_status_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_accept", detail: "was given a null out_error argument", } @@ -1361,21 +1428,58 @@ mod check_sampler_accept_status_tests { } #[test] - fn check_sampler_accept_status_maps_llama_cpp_out_of_memory() { + fn check_sampler_accept_status_reports_llama_cpp_running_out_of_memory() { let outcome = check_sampler_accept_status( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ); assert_eq!(outcome.err(), Some(SamplerAcceptError::LlamaCppOutOfMemory)); } + + #[test] + fn check_sampler_accept_status_exception_without_a_message_is_a_contract_error() { + assert_eq!( + check_sampler_accept_status( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ), + Err(SamplerAcceptError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_sampler_accept", + detail: "reported a thrown C++ exception without an error message", + } + ))) + ); + } + + #[test] + fn check_sampler_accept_status_preserves_an_unknown_status() { + assert_eq!( + check_sampler_accept_status( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_THREW_CXX_EXCEPTION + 1, + ptr::null_mut(), + ), + Err(SamplerAcceptError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_accept", + code: i64::from( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_ACCEPT_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1 + ), + })) + ); + } } #[cfg(test)] mod sampler_sample_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::sampler_sample_status_to_result; - use crate::SampleError; + use crate::error::sample_error::SampleError; #[test] fn sampler_sample_status_allocation_failure_maps_to_not_enough_memory() { @@ -1398,10 +1502,10 @@ mod sampler_sample_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_sample", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -1413,7 +1517,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( result, - Err(SampleError::FfiContract(crate::FfiContractError { + Err(SampleError::FfiContract(FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null ctx argument", })) @@ -1421,7 +1525,7 @@ mod sampler_sample_status_to_result_tests { } #[test] - fn sampler_sample_status_to_result_maps_null_sampler_arg() { + fn sampler_sample_status_to_result_reports_a_null_sampler_arg_as_a_contract_violation() { let outcome = sampler_sample_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_SAMPLER_ARG, 0, @@ -1430,7 +1534,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null sampler argument", } @@ -1440,7 +1544,7 @@ mod sampler_sample_status_to_result_tests { } #[test] - fn sampler_sample_status_to_result_maps_null_out_token_arg() { + fn sampler_sample_status_to_result_reports_a_null_out_token_arg_as_a_contract_violation() { let outcome = sampler_sample_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_TOKEN_ARG, 0, @@ -1449,7 +1553,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_token argument", } @@ -1459,7 +1563,7 @@ mod sampler_sample_status_to_result_tests { } #[test] - fn sampler_sample_status_to_result_maps_null_out_error_arg() { + fn sampler_sample_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { let outcome = sampler_sample_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_NULL_OUT_ERROR_ARG, 0, @@ -1468,7 +1572,7 @@ mod sampler_sample_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_sample", detail: "was given a null out_error argument", } @@ -1478,7 +1582,7 @@ mod sampler_sample_status_to_result_tests { } #[test] - fn sampler_sample_status_to_result_maps_llama_cpp_out_of_memory() { + fn sampler_sample_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = sampler_sample_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_LLAMA_CPP_OUT_OF_MEMORY, 0, @@ -1486,13 +1590,56 @@ mod sampler_sample_status_to_result_tests { ); assert_eq!(outcome.err(), Some(SampleError::LlamaCppOutOfMemory)); } + + #[test] + fn sampler_sample_status_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"sampler threw".as_ptr()) }; + + assert_eq!( + sampler_sample_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_LLAMA_CPP_THREW_CXX_EXCEPTION, + -1, + out_error, + ), + Err(SampleError::Reported { + message: "sampler threw".to_owned(), + }) + ); + } + + #[test] + fn sampler_sample_status_to_result_preserves_an_unknown_status() { + assert_eq!( + sampler_sample_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_LLAMA_CPP_THREW_CXX_EXCEPTION + 1, + -1, + ptr::null_mut(), + ), + Err(SampleError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_sample", + code: i64::from( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_SAMPLE_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1 + ), + })) + ); + } } #[cfg(test)] mod sampler_init_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_fault_injection::armed_fault::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::LlamaSampler; use super::sampler_init_status_to_result; + use crate::error::sampling_error::SamplingError; #[test] fn sampler_init_status_null_maps_to_sampler_unavailable() { @@ -1504,7 +1651,7 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::SamplingError::SamplerUnavailable { + Some(SamplingError::SamplerUnavailable { sampler: "penalties" }) ); @@ -1520,7 +1667,7 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::SamplingError::NotEnoughMemory) + Some(SamplingError::NotEnoughMemory) ); } @@ -1534,7 +1681,7 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::SamplingError::LlamaCppOutOfMemory { + Some(SamplingError::LlamaCppOutOfMemory { sampler: "penalties" }) ); @@ -1553,7 +1700,7 @@ mod sampler_init_status_to_result_tests { "penalties", ) .err(), - Some(crate::SamplingError::Reported { + Some(SamplingError::Reported { sampler: "penalties", message: "vector too long".to_owned(), }) @@ -1561,7 +1708,7 @@ mod sampler_init_status_to_result_tests { } #[test] - fn sampler_init_status_to_result_maps_null_out_sampler_arg() { + fn sampler_init_status_to_result_reports_a_null_out_sampler_arg_as_a_contract_violation() { assert_eq!( sampler_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_SAMPLER_ARG, @@ -1571,7 +1718,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_sampler argument", } @@ -1581,7 +1728,7 @@ mod sampler_init_status_to_result_tests { } #[test] - fn sampler_init_status_to_result_maps_null_out_error_arg() { + fn sampler_init_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { assert_eq!( sampler_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_NULL_OUT_ERROR_ARG, @@ -1591,7 +1738,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init", detail: "was given a null out_error argument", } @@ -1601,7 +1748,8 @@ mod sampler_init_status_to_result_tests { } #[test] - fn sampler_init_status_to_result_maps_llama_cpp_threw_cxx_exception_without_a_message() { + fn sampler_init_status_to_result_reports_an_exception_without_a_message_as_a_contract_violation() + { assert_eq!( sampler_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION, @@ -1611,17 +1759,17 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - crate::FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_init", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ) ); } #[test] - fn sampler_init_status_to_result_maps_an_unknown_status() { + fn sampler_init_status_to_result_preserves_an_unknown_status() { assert_eq!( sampler_init_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION + 1, @@ -1631,7 +1779,7 @@ mod sampler_init_status_to_result_tests { ) .err(), Some( - crate::FfiStatusError { + FfiStatusError { operation: "llama_rs_sampler_init", code: i64::from( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_LLAMA_CPP_THREW_CXX_EXCEPTION @@ -1642,13 +1790,50 @@ mod sampler_init_status_to_result_tests { ) ); } + + #[test] + fn initializing_a_selecting_sampler_reports_the_exception_llama_cpp_throws() { + let distribution_failure = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_DIST_THROWS, + ); + + LlamaSampler::dist(7).err() + }; + let greedy_failure = { + let _fault = ArmedFault::at( + llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_INIT_GREEDY_THROWS, + ); + + LlamaSampler::greedy().err() + }; + + assert_eq!( + [distribution_failure, greedy_failure], + [ + Some(SamplingError::Reported { + sampler: "distribution", + message: injected_fault_message(), + }), + Some(SamplingError::Reported { + sampler: "greedy", + message: injected_fault_message(), + }), + ] + ); + } } #[cfg(test)] mod sampler_chain_add_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::sampler_chain_add_status_to_result; + use crate::error::sampling_error::SamplingError; #[test] fn sampler_chain_add_status_allocation_failure_maps_to_not_enough_memory() { @@ -1657,7 +1842,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_ERROR_STRING_ALLOCATION_FAILED, ptr::null_mut(), ), - Err(crate::SamplingError::NotEnoughMemory) + Err(SamplingError::NotEnoughMemory) ); } @@ -1668,7 +1853,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ), - Err(crate::SamplingError::LlamaCppOutOfMemory { sampler: "chain" }) + Err(SamplingError::LlamaCppOutOfMemory { sampler: "chain" }) ); } @@ -1682,7 +1867,7 @@ mod sampler_chain_add_status_to_result_tests { llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, out_error, ), - Err(crate::SamplingError::Reported { + Err(SamplingError::Reported { sampler: "chain", message: "vector too long".to_owned(), }) @@ -1690,13 +1875,13 @@ mod sampler_chain_add_status_to_result_tests { } #[test] - fn sampler_chain_add_status_to_result_maps_null_chain_arg() { + fn sampler_chain_add_status_to_result_reports_a_null_chain_arg_as_a_contract_violation() { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_CHAIN_ARG, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null chain argument", } @@ -1705,13 +1890,13 @@ mod sampler_chain_add_status_to_result_tests { } #[test] - fn sampler_chain_add_status_to_result_maps_null_sampler_arg() { + fn sampler_chain_add_status_to_result_reports_a_null_sampler_arg_as_a_contract_violation() { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_SAMPLER_ARG, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null sampler argument", } @@ -1720,13 +1905,13 @@ mod sampler_chain_add_status_to_result_tests { } #[test] - fn sampler_chain_add_status_to_result_maps_null_out_error_arg() { + fn sampler_chain_add_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_NULL_OUT_ERROR_ARG, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "was given a null out_error argument", } @@ -1735,29 +1920,30 @@ mod sampler_chain_add_status_to_result_tests { } #[test] - fn sampler_chain_add_status_to_result_maps_llama_cpp_threw_cxx_exception_without_a_message() { + fn sampler_chain_add_status_to_result_reports_an_exception_without_a_message_as_a_contract_violation() + { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION, ptr::null_mut(), ), - Err(crate::FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_chain_add", detail: "reported a thrown C++ exception without an error message", - } + }) .into()) ); } #[test] - fn sampler_chain_add_status_to_result_maps_an_unknown_status() { + fn sampler_chain_add_status_to_result_preserves_an_unknown_status() { assert_eq!( sampler_chain_add_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION + 1, ptr::null_mut(), ), - Err(crate::FfiStatusError { + Err(FfiStatusError { operation: "llama_rs_sampler_chain_add", code: i64::from( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_CHAIN_ADD_LLAMA_CPP_THREW_CXX_EXCEPTION @@ -1771,10 +1957,14 @@ mod sampler_chain_add_status_to_result_tests { #[cfg(test)] mod sampler_init_grammar_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::sampler_init_grammar_status_to_result; - use crate::GrammarError; + use crate::error::grammar_error::GrammarError; #[test] fn sampler_init_grammar_status_null_maps_to_grammar_malformed() { @@ -1808,10 +1998,10 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } @@ -1824,7 +2014,7 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( result.unwrap_err(), - GrammarError::FfiContract(crate::FfiContractError { + GrammarError::FfiContract(FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_sampler argument", }) @@ -1832,7 +2022,8 @@ mod sampler_init_grammar_status_to_result_tests { } #[test] - fn sampler_init_grammar_status_to_result_maps_null_out_error_arg() { + fn sampler_init_grammar_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() + { let outcome = sampler_init_grammar_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_NULL_OUT_ERROR_ARG, ptr::null_mut(), @@ -1841,7 +2032,7 @@ mod sampler_init_grammar_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init_grammar", detail: "was given a null out_error argument", } @@ -1851,7 +2042,7 @@ mod sampler_init_grammar_status_to_result_tests { } #[test] - fn sampler_init_grammar_status_to_result_maps_llama_cpp_out_of_memory() { + fn sampler_init_grammar_status_to_result_reports_llama_cpp_running_out_of_memory() { let outcome = sampler_init_grammar_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), @@ -1859,14 +2050,56 @@ mod sampler_init_grammar_status_to_result_tests { ); assert_eq!(outcome.err(), Some(GrammarError::LlamaCppOutOfMemory)); } + + #[test] + fn sampler_init_grammar_status_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"grammar threw".as_ptr()) }; + + assert_eq!( + sampler_init_grammar_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + .err(), + Some(GrammarError::Reported { + message: "grammar threw".to_owned(), + }) + ); + } + + #[test] + fn sampler_init_grammar_status_to_result_preserves_an_unknown_status() { + assert_eq!( + sampler_init_grammar_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1, + ptr::null_mut(), + ptr::null_mut(), + ) + .err(), + Some(GrammarError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_init_grammar", + code: i64::from( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1 + ), + })) + ); + } } #[cfg(test)] mod sampler_init_grammar_lazy_patterns_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ptr; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + use super::sampler_init_grammar_lazy_patterns_status_to_result; - use crate::GrammarError; + use crate::error::grammar_error::GrammarError; #[test] fn sampler_init_grammar_lazy_patterns_status_null_maps_to_lazy_patterns_grammar_malformed() { @@ -1900,14 +2133,33 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( result.unwrap_err(), - crate::FfiContractError { + CppStringError::Missing(FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "reported a thrown C++ exception without an error message", - } + }) .into() ); } + #[test] + fn an_invalid_trigger_pattern_without_a_message_is_a_contract_error() { + assert_eq!( + sampler_init_grammar_lazy_patterns_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_INVALID_TRIGGER_PATTERN, + ptr::null_mut(), + ptr::null_mut(), + ) + .err(), + Some( + CppStringError::Missing(FfiContractError { + operation: "llama_rs_sampler_init_grammar_lazy_patterns", + detail: "reported a thrown C++ exception without an error message", + }) + .into() + ) + ); + } + #[test] fn lazy_grammar_null_output_argument_status_is_a_contract_error() { let status = llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_SAMPLER_ARG; @@ -1919,7 +2171,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( result.unwrap_err(), - GrammarError::FfiContract(crate::FfiContractError { + GrammarError::FfiContract(FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_sampler argument", }) @@ -1927,7 +2179,8 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { } #[test] - fn sampler_init_grammar_lazy_patterns_status_to_result_maps_null_out_error_arg() { + fn sampler_init_grammar_lazy_patterns_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() + { let outcome = sampler_init_grammar_lazy_patterns_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_NULL_OUT_ERROR_ARG, ptr::null_mut(), @@ -1936,7 +2189,7 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { assert_eq!( outcome.err(), Some( - crate::FfiContractError { + FfiContractError { operation: "llama_rs_sampler_init_grammar_lazy_patterns", detail: "was given a null out_error argument", } @@ -1946,8 +2199,255 @@ mod sampler_init_grammar_lazy_patterns_status_to_result_tests { } #[test] - fn sampler_init_grammar_lazy_patterns_status_to_result_maps_llama_cpp_out_of_memory() { + fn sampler_init_grammar_lazy_patterns_status_to_result_reports_llama_cpp_running_out_of_memory() + { let outcome = sampler_init_grammar_lazy_patterns_status_to_result(llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_OUT_OF_MEMORY, ptr::null_mut(), ptr::null_mut()); assert_eq!(outcome.err(), Some(GrammarError::LlamaCppOutOfMemory)); } + + #[test] + fn sampler_init_grammar_lazy_patterns_status_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"lazy grammar threw".as_ptr()) }; + + assert_eq!( + sampler_init_grammar_lazy_patterns_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + out_error, + ) + .err(), + Some(GrammarError::Reported { + message: "lazy grammar threw".to_owned(), + }) + ); + } + + #[test] + fn sampler_init_grammar_lazy_patterns_status_to_result_preserves_an_unknown_status() { + let unknown_status = + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_INIT_GRAMMAR_LAZY_PATTERNS_LLAMA_CPP_THREW_CXX_EXCEPTION + + 1; + + assert_eq!( + sampler_init_grammar_lazy_patterns_status_to_result( + unknown_status, + ptr::null_mut(), + ptr::null_mut(), + ) + .err(), + Some(GrammarError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_init_grammar_lazy_patterns", + code: i64::from(unknown_status), + })) + ); + } +} + +#[cfg(test)] +mod sampler_application_tests { + use std::ptr; + + use llama_cpp_fault_injection::armed_fault::ArmedFault; + use llama_cpp_fault_injection::injected_fault_message::injected_fault_message; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + + use super::LlamaSampler; + use crate::error::sample_error::SampleError; + use crate::error::sampler_apply_error::SamplerApplyError; + use crate::token::LlamaToken; + use crate::token::data::LlamaTokenData; + use crate::token::data_array::LlamaTokenDataArray; + + fn excluded_candidates() -> LlamaTokenDataArray { + LlamaTokenDataArray::new( + vec![ + LlamaTokenData::new(LlamaToken::new(0), f32::NEG_INFINITY, 0.0), + LlamaTokenData::new(LlamaToken::new(1), f32::NEG_INFINITY, 0.0), + ], + false, + ) + } + + unsafe extern "C" fn claim_more_candidates_than_exist( + _sampler: *mut llama_cpp_bindings_sys::llama_sampler, + candidates: *mut llama_cpp_bindings_sys::llama_token_data_array, + ) { + unsafe { (*candidates).size = usize::MAX }; + } + + static mut OVERREPORTING_SAMPLER: llama_cpp_bindings_sys::llama_sampler_i = + llama_cpp_bindings_sys::llama_sampler_i { + name: None, + accept: None, + apply: Some(claim_more_candidates_than_exist), + reset: None, + clone: None, + free: None, + backend_init: None, + backend_accept: None, + backend_apply: None, + backend_set_input: None, + backend_reset: None, + copy_state: None, + }; + + #[test] + fn a_sampler_claiming_more_candidates_than_exist_is_a_contract_violation() { + let overreporting_sampler = + LlamaSampler::initialize("overreporting", |out_sampler, out_error| unsafe { + llama_cpp_bindings_sys::llama_rs_sampler_init( + &raw mut OVERREPORTING_SAMPLER, + ptr::null_mut(), + out_sampler, + out_error, + ) + }) + .unwrap(); + + assert_eq!( + overreporting_sampler.apply(&mut excluded_candidates()), + Err(SampleError::SamplerApply(SamplerApplyError::FfiContract( + FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the llama.cpp sampler grew the token data array beyond its capacity", + } + ))) + ); + } + + #[test] + fn an_exception_llama_cpp_throws_while_applying_a_sampler_is_reported() { + let greedy_sampler = LlamaSampler::greedy().unwrap(); + let _fault = + ArmedFault::at(llama_cpp_bindings_sys::LLAMA_RS_FAULT_POINT_SAMPLER_APPLY_THROWS); + + assert_eq!( + greedy_sampler.apply(&mut excluded_candidates()), + Err(SampleError::SamplerApply(SamplerApplyError::Reported { + message: injected_fault_message(), + })) + ); + } +} + +#[cfg(test)] +mod sampler_apply_status_to_result_tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; + use std::ptr; + + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + + use super::sampler_apply_status_to_result; + use crate::error::sampler_apply_error::SamplerApplyError; + + #[test] + fn sampler_apply_status_allocation_failed_returns_not_enough_memory() { + assert_eq!( + sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED, + ptr::null_mut(), + ), + Err(SamplerApplyError::NotEnoughMemory), + ); + } + + #[test] + fn sampler_apply_status_cxx_exception_without_a_message_is_a_contract_error() { + assert_eq!( + sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, + ptr::null_mut(), + ), + Err(CppStringError::Missing(FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "reported a thrown C++ exception without an error message", + }) + .into()), + ); + } + + #[test] + fn sampler_apply_unknown_status_is_preserved() { + let result = sampler_apply_status_to_result(255, ptr::null_mut()); + + assert_eq!( + result, + Err(SamplerApplyError::FfiStatus(FfiStatusError { + operation: "llama_rs_sampler_apply", + code: 255, + })) + ); + } + + #[test] + fn sampler_apply_status_to_result_reports_a_null_sampler_arg_as_a_null_sampler() { + let outcome = sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_SAMPLER_ARG, + ptr::null_mut(), + ); + assert_eq!(outcome.err(), Some(SamplerApplyError::NullSampler)); + } + + #[test] + fn sampler_apply_status_to_result_reports_a_null_data_array_arg_as_a_contract_violation() { + let outcome = sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "was given a null data_array argument", + } + .into() + ) + ); + } + + #[test] + fn sampler_apply_status_to_result_reports_a_null_out_error_arg_as_a_contract_violation() { + let outcome = sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG, + ptr::null_mut(), + ); + assert_eq!( + outcome.err(), + Some( + FfiContractError { + operation: "llama_rs_sampler_apply", + detail: "was given a null out_error argument", + } + .into() + ) + ); + } + + #[test] + fn sampler_apply_status_to_result_reports_llama_cpp_running_out_of_memory() { + let outcome = sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_OUT_OF_MEMORY, + ptr::null_mut(), + ); + assert_eq!(outcome.err(), Some(SamplerApplyError::LlamaCppOutOfMemory)); + } + + #[test] + fn sampler_apply_status_cxx_exception_carries_its_message() { + let out_error = + unsafe { llama_cpp_bindings_sys::llama_rs_string_dup(c"apply threw".as_ptr()) }; + + assert_eq!( + sampler_apply_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, + out_error, + ), + Err(SamplerApplyError::Reported { + message: "apply threw".to_owned(), + }) + ); + } } diff --git a/llama-cpp-bindings/src/send_logs_to_log.rs b/llama-cpp-bindings/src/send_logs_to_log.rs index 9c5fa7e19..285339685 100644 --- a/llama-cpp-bindings/src/send_logs_to_log.rs +++ b/llama-cpp-bindings/src/send_logs_to_log.rs @@ -8,8 +8,15 @@ ) )] +use std::ffi::CStr; +use std::ffi::c_char; +use std::ffi::c_void; +use std::sync::PoisonError; use std::sync::{Mutex, OnceLock}; +use log::log; +use log::warn; + use llama_cpp_log_decoder::decode_anomaly::DecodeAnomaly; use llama_cpp_log_decoder::decode_output::DecodeOutput; use llama_cpp_log_decoder::incoming_log_level::IncomingLogLevel; @@ -23,7 +30,7 @@ use crate::log_record::LogRecord; struct LogSource { decoder: Mutex, target: &'static str, - options: LogOptions, + options: Mutex, } impl LogSource { @@ -31,9 +38,20 @@ impl LogSource { Self { decoder: Mutex::new(LogDecoder::new()), target, - options, + options: Mutex::new(options), } } + + fn options(&self) -> LogOptions { + self.options + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + fn replace_options(&self, options: LogOptions) { + *self.options.lock().unwrap_or_else(PoisonError::into_inner) = options; + } } static LLAMA_SOURCE: OnceLock = OnceLock::new(); @@ -84,8 +102,8 @@ fn resolve_record(line: LogLine, demote_info_to_debug: bool) -> LogRecord { } fn dispatch_line(source: &LogSource, line: LogLine) { - let LogRecord { level, text } = resolve_record(line, source.options.demote_info_to_debug); - log::log!(target: source.target, level, "{text}"); + let LogRecord { level, text } = resolve_record(line, source.options().demote_info_to_debug); + log!(target: source.target, level, "{text}"); } fn dispatch_output(source: &LogSource, output: DecodeOutput) { @@ -100,7 +118,7 @@ fn dispatch_output(source: &LogSource, output: DecodeOutput) { } fn dispatch_anomaly(source: &LogSource, anomaly: DecodeAnomaly) { - log::warn!( + warn!( target: source.target, "llama.cpp log decoder anomaly: {anomaly:?}", ); @@ -108,24 +126,24 @@ fn dispatch_anomaly(source: &LogSource, anomaly: DecodeAnomaly) { unsafe extern "C" fn logs_to_log( raw_level: llama_cpp_bindings_sys::ggml_log_level, - text_ptr: *const std::os::raw::c_char, - data_ptr: *mut std::os::raw::c_void, + text_ptr: *const c_char, + data_ptr: *mut c_void, ) { let source: &LogSource = unsafe { &*data_ptr.cast::() }; - if source.options.disabled { + if source.options().disabled { return; } if text_ptr.is_null() { - log::warn!( + warn!( target: source.target, "received NULL text pointer from llama.cpp log callback", ); return; } - let text_cstr = unsafe { std::ffi::CStr::from_ptr(text_ptr) }; + let text_cstr = unsafe { CStr::from_ptr(text_ptr) }; let text = text_cstr.to_string_lossy(); let incoming = ggml_level_to_incoming(raw_level); @@ -134,7 +152,7 @@ unsafe extern "C" fn logs_to_log( let mut decoder = source .decoder .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); decoder.feed(incoming, &text) }; @@ -146,41 +164,56 @@ unsafe extern "C" fn logs_to_log( } pub fn send_logs_to_log(options: LogOptions) { - let llama_source: *const LogSource = - LLAMA_SOURCE.get_or_init(|| LogSource::new("llama.cpp", options.clone())); - let ggml_source: *const LogSource = GGML_SOURCE.get_or_init(|| LogSource::new("ggml", options)); + let llama_source = + LLAMA_SOURCE.get_or_init(|| LogSource::new("llama.cpp", LogOptions::default())); + let ggml_source = GGML_SOURCE.get_or_init(|| LogSource::new("ggml", LogOptions::default())); + + llama_source.replace_options(options.clone()); + ggml_source.replace_options(options); + + let llama_source: *const LogSource = llama_source; + let ggml_source: *const LogSource = ggml_source; unsafe { llama_cpp_bindings_sys::llama_log_set( Some(logs_to_log), - llama_source.cast::().cast_mut(), + llama_source.cast::().cast_mut(), ); llama_cpp_bindings_sys::ggml_log_set( Some(logs_to_log), - ggml_source.cast::().cast_mut(), + ggml_source.cast::().cast_mut(), ); } } #[cfg(test)] mod tests { - use crate::log_record::LogRecord; + use std::ffi::CStr; + use std::ffi::c_void; + use std::ptr; + use std::sync::PoisonError; use std::sync::{Mutex, Once}; + use std::thread; + + use log::{Level, Log, Metadata, Record}; + use serial_test::serial; use llama_cpp_log_decoder::decode_output::DecodeOutput; use llama_cpp_log_decoder::incoming_log_level::IncomingLogLevel; use llama_cpp_log_decoder::log_level::LogLevel; use llama_cpp_log_decoder::log_line::LogLine; - use log::{Level, Log, Metadata, Record}; - use serial_test::serial; - use super::{ - GGML_SOURCE, LLAMA_SOURCE, LogSource, dispatch_output, ggml_level_to_incoming, logs_to_log, - resolve_record, send_logs_to_log, - }; + use super::LLAMA_SOURCE; + use super::LogSource; + use super::dispatch_output; + use super::ggml_level_to_incoming; + use super::logs_to_log; + use super::resolve_record; use crate::log_options::LogOptions; + use crate::log_record::LogRecord; + use crate::send_logs_to_log::send_logs_to_log; - #[derive(Clone, Debug)] + #[derive(Debug)] struct CapturedRecord { level: Level, target: String, @@ -197,10 +230,7 @@ mod tests { } fn log(&self, record: &Record) { - let mut guard = self - .records - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut guard = self.records.lock().unwrap_or_else(PoisonError::into_inner); guard.push(CapturedRecord { level: record.level(), target: record.target().to_owned(), @@ -218,39 +248,46 @@ mod tests { fn ensure_test_logger_installed() { INSTALL.call_once(|| { - if log::set_logger(&TEST_LOGGER).is_ok() { - log::set_max_level(log::LevelFilter::Trace); - } + log::set_logger(&TEST_LOGGER).unwrap(); + log::set_max_level(log::LevelFilter::Trace); }); } - fn records_for(target: &str) -> Vec { + fn records_for(target: &str) -> Vec { let guard = TEST_LOGGER .records .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); guard .iter() .filter(|record| record.target == target) - .cloned() + .map(|record| LogRecord { + level: record.level, + text: record.message.clone(), + }) + .collect() + } + + fn records_with_text(target: &str, text: &str) -> Vec { + records_for(target) + .into_iter() + .filter(|record| record.text == text) .collect() } fn invoke_callback( level: llama_cpp_bindings_sys::ggml_log_level, - text: &std::ffi::CStr, + text: &CStr, source: &LogSource, ) { - let ptr = std::ptr::from_ref(source) - .cast::() - .cast_mut(); + let ptr = ptr::from_ref(source).cast::().cast_mut(); unsafe { logs_to_log(level, text.as_ptr(), ptr); } } #[test] - fn test_logger_enabled_and_flush() { + fn the_capturing_logger_enables_every_record() { let metadata = Metadata::builder() .level(Level::Info) .target("test-logger-enabled") @@ -261,7 +298,7 @@ mod tests { } #[test] - fn ggml_level_to_incoming_known_constants() { + fn every_ggml_log_level_maps_to_its_incoming_level() { assert_eq!( ggml_level_to_incoming(llama_cpp_bindings_sys::GGML_LOG_LEVEL_NONE), IncomingLogLevel::None, @@ -289,7 +326,7 @@ mod tests { } #[test] - fn ggml_level_to_incoming_unknown_value() { + fn an_unknown_ggml_log_level_keeps_its_raw_value() { assert_eq!( ggml_level_to_incoming(9999), IncomingLogLevel::Unknown(9999) @@ -297,7 +334,7 @@ mod tests { } #[test] - fn dispatch_when_disabled() { + fn disabled_logs_emit_no_record() { ensure_test_logger_installed(); let target = "test-dispatch-when-disabled"; @@ -308,11 +345,11 @@ mod tests { &source, ); - assert!(records_for(target).is_empty()); + assert_eq!(records_for(target), Vec::::new()); } #[test] - fn demote_info_to_debug_on_info() { + fn demoting_info_logs_emits_info_as_debug() { ensure_test_logger_installed(); let target = "test-demote-info-on-info"; @@ -326,13 +363,17 @@ mod tests { &source, ); - assert!(records_for(target).iter().any(|record| { - record.level == Level::Debug && record.message.contains("info-line") - })); + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Debug, + text: "info-line".to_owned(), + }] + ); } #[test] - fn demote_info_to_debug_on_warn() { + fn demoting_info_logs_keeps_warnings_as_warnings() { ensure_test_logger_installed(); let target = "test-demote-info-on-warn"; @@ -346,30 +387,34 @@ mod tests { &source, ); - assert!( - records_for(target).iter().any(|record| { - record.level == Level::Warn && record.message.contains("warn-line") - }) + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Warn, + text: "warn-line".to_owned(), + }] ); } #[test] - fn dispatch_unknown_level() { + fn an_unknown_level_is_logged_as_a_warning_naming_it() { ensure_test_logger_installed(); let target = "test-dispatch-unknown-level"; let source = LogSource::new(target, LogOptions::default()); invoke_callback(9999, c"weird\n", &source); - assert!(records_for(target).iter().any(|record| { - record.level == Level::Warn - && record.message.contains("[unknown level 9999]") - && record.message.contains("weird") - })); + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Warn, + text: "[unknown level 9999] weird".to_owned(), + }] + ); } #[test] - fn dispatch_orphan_cont_anomaly() { + fn an_orphan_continuation_is_logged_with_an_anomaly_warning() { ensure_test_logger_installed(); let target = "test-dispatch-orphan-cont"; @@ -380,9 +425,19 @@ mod tests { &source, ); - assert!(records_for(target).iter().any(|record| { - record.level == Level::Warn && record.message.contains("OrphanCont") - })); + assert_eq!( + records_for(target), + [ + LogRecord { + level: Level::Info, + text: "ghost".to_owned(), + }, + LogRecord { + level: Level::Warn, + text: "llama.cpp log decoder anomaly: OrphanCont".to_owned(), + }, + ] + ); } #[test] @@ -410,7 +465,7 @@ mod tests { let source = LogSource::new(target, LogOptions::default()); dispatch_output(&source, DecodeOutput::None); - assert!(records_for(target).is_empty()); + assert_eq!(records_for(target), Vec::::new()); } #[test] @@ -433,64 +488,115 @@ mod tests { }, ); - let records = records_for(target); - assert!( - records - .iter() - .any(|record| record.message.contains("earlier-line")) + assert_eq!( + records_for(target), + [ + LogRecord { + level: Level::Info, + text: "earlier-line".to_owned(), + }, + LogRecord { + level: Level::Warn, + text: "current-line".to_owned(), + }, + ] ); - assert!( - records - .iter() - .any(|record| record.message.contains("current-line")) + } + + #[test] + #[serial] + fn sending_logs_again_applies_the_new_options() { + ensure_test_logger_installed(); + send_logs_to_log(LogOptions::default()); + send_logs_to_log(LogOptions::default().with_logs_enabled(false)); + + let mut ggml_callback: llama_cpp_bindings_sys::ggml_log_callback = None; + let mut ggml_user_data: *mut c_void = ptr::null_mut(); + + unsafe { + llama_cpp_bindings_sys::ggml_log_get(&raw mut ggml_callback, &raw mut ggml_user_data); + ggml_callback.unwrap()( + llama_cpp_bindings_sys::GGML_LOG_LEVEL_WARN, + c"silenced by the second call\n".as_ptr(), + ggml_user_data, + ); + } + send_logs_to_log(LogOptions::default()); + + assert_eq!( + records_with_text("ggml", "silenced by the second call"), + Vec::::new() ); } #[test] #[serial] - fn send_logs_to_log_initialization() { + fn sending_logs_to_log_creates_the_llama_cpp_source_and_routes_ggml_logs() { ensure_test_logger_installed(); send_logs_to_log(LogOptions::default()); - assert!(LLAMA_SOURCE.get().is_some()); - assert!(GGML_SOURCE.get().is_some()); + let mut ggml_callback: llama_cpp_bindings_sys::ggml_log_callback = None; + let mut ggml_user_data: *mut c_void = ptr::null_mut(); + + unsafe { + llama_cpp_bindings_sys::ggml_log_get(&raw mut ggml_callback, &raw mut ggml_user_data); + ggml_callback.unwrap()( + llama_cpp_bindings_sys::GGML_LOG_LEVEL_WARN, + c"routed through ggml\n".as_ptr(), + ggml_user_data, + ); + } + + assert_eq!( + LLAMA_SOURCE.get().map(|llama_source| llama_source.target), + Some("llama.cpp") + ); + assert_eq!( + records_with_text("ggml", "routed through ggml"), + [LogRecord { + level: Level::Warn, + text: "routed through ggml".to_owned(), + }] + ); } #[test] - fn null_text_pointer() { + fn a_null_text_pointer_is_logged_as_a_warning() { ensure_test_logger_installed(); let target = "test-null-text-pointer"; let source = LogSource::new(target, LogOptions::default()); - let source_ptr = std::ptr::from_ref(&source) - .cast::() - .cast_mut(); + let source_ptr = ptr::from_ref(&source).cast::().cast_mut(); unsafe { logs_to_log( llama_cpp_bindings_sys::GGML_LOG_LEVEL_INFO, - std::ptr::null(), + ptr::null(), source_ptr, ); } - assert!(records_for(target).iter().any(|record| { - record.level == Level::Warn && record.message.contains("NULL text pointer") - })); + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Warn, + text: "received NULL text pointer from llama.cpp log callback".to_owned(), + }] + ); } #[test] - fn decoder_mutex_poison() { + fn a_poisoned_decoder_still_decodes_logs() { ensure_test_logger_installed(); let target = "test-decoder-mutex-poison"; let source = LogSource::new(target, LogOptions::default()); - std::thread::scope(|scope| { + thread::scope(|scope| { let handle = scope.spawn(|| { let _guard = source .decoder .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); panic!("intentional poison"); }); let _ = handle.join(); @@ -504,10 +610,12 @@ mod tests { &source, ); - assert!( - records_for(target) - .iter() - .any(|record| record.message.contains("after-poison")) + assert_eq!( + records_for(target), + [LogRecord { + level: Level::Info, + text: "after-poison".to_owned(), + }] ); } } diff --git a/llama-cpp-bindings/src/streaming_json_probe.rs b/llama-cpp-bindings/src/streaming_json_probe.rs deleted file mode 100644 index 816398f8c..000000000 --- a/llama-cpp-bindings/src/streaming_json_probe.rs +++ /dev/null @@ -1,599 +0,0 @@ -use std::mem; - -use serde::Deserialize; -use serde::de::IgnoredAny; - -use crate::json_probe_outcome::JsonProbeOutcome; - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct BareJsonToolCall { - name: String, - #[serde(rename = "arguments")] - _arguments: Option, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ToolCallField { - Arguments, - Name, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -enum ProbeState { - AwaitingObjectOpen, - AwaitingFirstKeyOrClose, - AwaitingKey, - InKey { - quoted_key: String, - escaped: bool, - }, - AwaitingColon(ToolCallField), - AwaitingValue(ToolCallField), - InName { - escaped: bool, - }, - InArguments { - depth: usize, - in_string: bool, - escaped: bool, - }, - AwaitingCommaOrClose, - Closed, - Failed, -} - -const fn is_json_whitespace(character: char) -> bool { - matches!(character, ' ' | '\t' | '\n' | '\r') -} - -fn field_named(quoted_key: &str) -> Option { - match serde_json::from_str::(quoted_key).ok()?.as_str() { - "arguments" => Some(ToolCallField::Arguments), - "name" => Some(ToolCallField::Name), - _ => None, - } -} - -fn is_named_tool_call(held_text: &str) -> bool { - serde_json::from_str::(held_text) - .is_ok_and(|tool_call| !tool_call.name.is_empty()) -} - -impl ProbeState { - fn advance(self, character: char) -> Self { - match self { - Self::AwaitingObjectOpen => match character { - '{' => Self::AwaitingFirstKeyOrClose, - _ if is_json_whitespace(character) => Self::AwaitingObjectOpen, - _ => Self::Failed, - }, - Self::AwaitingFirstKeyOrClose => match character { - '"' => Self::InKey { - quoted_key: String::from('"'), - escaped: false, - }, - '}' => Self::Closed, - _ if is_json_whitespace(character) => Self::AwaitingFirstKeyOrClose, - _ => Self::Failed, - }, - Self::AwaitingKey => match character { - '"' => Self::InKey { - quoted_key: String::from('"'), - escaped: false, - }, - _ if is_json_whitespace(character) => Self::AwaitingKey, - _ => Self::Failed, - }, - Self::InKey { - mut quoted_key, - escaped, - } => { - quoted_key.push(character); - - if escaped || character != '"' { - Self::InKey { - quoted_key, - escaped: !escaped && character == '\\', - } - } else { - field_named("ed_key).map_or(Self::Failed, Self::AwaitingColon) - } - } - Self::AwaitingColon(field) => match character { - ':' => Self::AwaitingValue(field), - _ if is_json_whitespace(character) => Self::AwaitingColon(field), - _ => Self::Failed, - }, - Self::AwaitingValue(field) => match (field, character) { - (ToolCallField::Name, '"') => Self::InName { escaped: false }, - (ToolCallField::Arguments, '{') => Self::InArguments { - depth: 1, - in_string: false, - escaped: false, - }, - _ if is_json_whitespace(character) => Self::AwaitingValue(field), - _ => Self::Failed, - }, - Self::InName { escaped } => { - if !escaped && character == '"' { - Self::AwaitingCommaOrClose - } else { - Self::InName { - escaped: !escaped && character == '\\', - } - } - } - Self::InArguments { - depth, - in_string, - escaped, - } => Self::advance_arguments(depth, in_string, escaped, character), - Self::AwaitingCommaOrClose => match character { - ',' => Self::AwaitingKey, - '}' => Self::Closed, - _ if is_json_whitespace(character) => Self::AwaitingCommaOrClose, - _ => Self::Failed, - }, - Self::Closed if is_json_whitespace(character) => Self::Closed, - Self::Closed | Self::Failed => Self::Failed, - } - } - - const fn advance_arguments( - depth: usize, - in_string: bool, - escaped: bool, - character: char, - ) -> Self { - if in_string { - return Self::InArguments { - depth, - in_string: escaped || character != '"', - escaped: !escaped && character == '\\', - }; - } - - match character { - '"' => Self::InArguments { - depth, - in_string: true, - escaped: false, - }, - '{' | '[' => Self::InArguments { - depth: depth + 1, - in_string: false, - escaped: false, - }, - '}' | ']' if depth == 1 => Self::AwaitingCommaOrClose, - '}' | ']' => Self::InArguments { - depth: depth - 1, - in_string: false, - escaped: false, - }, - _ => Self::InArguments { - depth, - in_string: false, - escaped: false, - }, - } - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct StreamingJsonProbe { - held_text: String, - state: ProbeState, -} - -impl Default for StreamingJsonProbe { - fn default() -> Self { - Self { - held_text: String::new(), - state: ProbeState::AwaitingObjectOpen, - } - } -} - -impl StreamingJsonProbe { - pub fn feed(&mut self, piece: &str) -> JsonProbeOutcome { - self.held_text.push_str(piece); - - for character in piece.chars() { - let state = mem::replace(&mut self.state, ProbeState::Failed); - - self.state = state.advance(character); - - if self.state == ProbeState::Failed { - return JsonProbeOutcome::Failed; - } - } - - match self.state { - ProbeState::Closed if is_named_tool_call(&self.held_text) => { - JsonProbeOutcome::CompletedValid - } - ProbeState::Closed | ProbeState::Failed => JsonProbeOutcome::Failed, - _ => JsonProbeOutcome::StillPossiblyValid, - } - } -} - -#[cfg(test)] -mod tests { - use super::StreamingJsonProbe; - use crate::json_probe_outcome::JsonProbeOutcome; - - fn probe(buffer: &str) -> JsonProbeOutcome { - StreamingJsonProbe::default().feed(buffer) - } - - #[test] - fn empty_buffer_is_still_possibly_valid() { - assert_eq!(probe(""), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn whitespace_only_buffer_is_still_possibly_valid() { - assert_eq!(probe(" \n "), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn single_open_brace_is_still_possibly_valid() { - assert_eq!(probe("{"), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn open_brace_with_trailing_space_is_still_possibly_valid() { - assert_eq!(probe("{ "), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn open_brace_with_quote_starting_key_is_still_possibly_valid() { - assert_eq!(probe(r#"{ ""#), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn partial_name_key_is_still_possibly_valid() { - assert_eq!(probe(r#"{ "name""#), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn partial_name_value_quote_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": ""#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn partial_name_value_letters_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "ge"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn complete_name_string_no_comma_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather""#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn name_then_comma_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather","#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn name_then_partial_arguments_key_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "argum"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn name_then_arguments_key_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "arguments""#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn name_then_arguments_open_brace_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "arguments": {"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn arguments_with_partial_inner_key_value_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "arguments": {"location":"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn arguments_with_partial_inner_string_value_is_still_possibly_valid() { - assert_eq!( - probe(r#"{ "name": "get_weather", "arguments": {"location": "Pa"#), - JsonProbeOutcome::StillPossiblyValid, - ); - } - - #[test] - fn complete_simple_tool_call_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_internal_whitespace_is_completed_valid() { - assert_eq!( - probe(r#"{"name": "f", "arguments": {}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_string_argument_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"get_weather","arguments":{"location":"Paris"}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_multiple_arguments_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"book_flight","arguments":{"from":"NYC","to":"PAR","passengers":2}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_nested_arguments_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{"a":{"b":[1,2,3]}}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_close_brace_inside_string_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{"q":"a } b"}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_escaped_quotes_in_string_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{"q":"he said \"hi\""}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_unicode_strings_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"日本語","arguments":{"city":"パリ"}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_trailing_whitespace_is_completed_valid() { - assert_eq!( - probe("{\"name\":\"f\",\"arguments\":{}}\n"), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_with_array_inside_arguments_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"f","arguments":{"items":[1,2,3]}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_without_arguments_field_is_completed_valid() { - assert_eq!( - probe(r#"{"name":"ping"}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn top_level_array_is_failed() { - assert_eq!(probe("["), JsonProbeOutcome::Failed); - } - - #[test] - fn top_level_scalar_number_is_failed() { - assert_eq!(probe("123"), JsonProbeOutcome::Failed); - } - - #[test] - fn top_level_string_is_failed() { - assert_eq!(probe(r#""hi""#), JsonProbeOutcome::Failed); - } - - #[test] - fn complete_object_with_wrong_first_key_is_failed() { - assert_eq!(probe(r#"{"foo":"bar"}"#), JsonProbeOutcome::Failed,); - } - - #[test] - fn complete_object_with_non_string_name_is_failed() { - assert_eq!( - probe(r#"{"name":123,"arguments":{}}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn complete_object_with_null_name_is_failed() { - assert_eq!( - probe(r#"{"name":null,"arguments":{}}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn complete_object_with_arguments_as_array_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":[]}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn complete_object_with_arguments_as_string_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":"hi"}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn complete_object_with_third_top_level_key_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":{},"extra":1}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn complete_object_with_empty_name_is_failed() { - assert_eq!( - probe(r#"{"name":"","arguments":{}}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn complete_object_with_trailing_garbage_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":{}}garbage"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn empty_object_is_failed_due_to_missing_required_name() { - assert_eq!(probe("{}"), JsonProbeOutcome::Failed); - } - - #[test] - fn complete_object_with_arguments_only_no_name_is_failed() { - assert_eq!(probe(r#"{"arguments":{}}"#), JsonProbeOutcome::Failed,); - } - - #[test] - fn leading_whitespace_then_open_brace_is_still_possibly_valid() { - assert_eq!(probe("\n \n{"), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn leading_whitespace_then_complete_tool_call_is_completed_valid() { - assert_eq!( - probe("\n {\"name\":\"f\",\"arguments\":{}}"), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn complete_tool_call_followed_by_second_object_is_failed() { - assert_eq!( - probe(r#"{"name":"a","arguments":{}}{"name":"b","arguments":{}}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn buffer_with_only_open_quote_is_still_possibly_valid() { - assert_eq!(probe(r#"{ "n"#), JsonProbeOutcome::StillPossiblyValid,); - } - - #[test] - fn buffer_with_complete_first_field_unknown_second_key_is_failed() { - assert_eq!( - probe(r#"{ "name": "f", "foo": 1}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn unicode_letter_inside_name_value_completes_validly() { - assert_eq!( - probe(r#"{"name":"éclair","arguments":{}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn arguments_field_with_explicit_null_is_failed() { - assert_eq!( - probe(r#"{"name":"f","arguments":null}"#), - JsonProbeOutcome::Failed, - ); - } - - #[test] - fn syntactically_malformed_object_is_failed() { - assert_eq!(probe("{,}"), JsonProbeOutcome::Failed,); - } - - #[test] - fn key_written_with_an_escape_sequence_is_recognized() { - assert_eq!( - probe(r#"{"na\u006de":"f","arguments":{}}"#), - JsonProbeOutcome::CompletedValid, - ); - } - - #[test] - fn tool_call_fed_one_character_at_a_time_completes_on_its_last_character() { - let tool_call = r#"{"name":"f","arguments":{"q":"a } b"}}"#; - let mut streaming_probe = StreamingJsonProbe::default(); - let mut outcomes = Vec::new(); - - for character in tool_call.chars() { - outcomes.push(streaming_probe.feed(&character.to_string())); - } - - let (last_outcome, earlier_outcomes) = outcomes - .split_last() - .expect("the tool call must produce outcomes"); - - assert_eq!(*last_outcome, JsonProbeOutcome::CompletedValid); - assert!( - earlier_outcomes - .iter() - .all(|outcome| *outcome == JsonProbeOutcome::StillPossiblyValid) - ); - } - - #[test] - fn syntax_error_inside_arguments_fails_when_the_object_closes() { - assert_eq!( - probe(r#"{"name":"f","arguments":{"q" 1}}"#), - JsonProbeOutcome::Failed, - ); - } -} diff --git a/llama-cpp-bindings/src/streaming_marker.rs b/llama-cpp-bindings/src/streaming_marker.rs deleted file mode 100644 index f2706d6be..000000000 --- a/llama-cpp-bindings/src/streaming_marker.rs +++ /dev/null @@ -1,91 +0,0 @@ -use crate::marker_role::MarkerRole; -use crate::sampled_token_section::SampledTokenSection; -use crate::token::LlamaToken; - -#[derive(Clone, Debug, Eq, PartialEq)] -/// A normalized token sequence and every semantic role attached to it. -pub struct StreamingMarker { - tokens: Vec, - roles: Vec, -} - -impl StreamingMarker { - #[must_use] - pub fn new(tokens: Vec, role: MarkerRole) -> Self { - Self { - tokens, - roles: vec![role], - } - } - - pub fn add_role(&mut self, role: MarkerRole) { - if !self.roles.contains(&role) { - self.roles.push(role); - } - } - - #[must_use] - /// Returns the tokens that form this marker. - pub fn tokens(&self) -> &[LlamaToken] { - &self.tokens - } - - #[must_use] - /// Returns the transitions associated with this marker. - pub fn roles(&self) -> &[MarkerRole] { - &self.roles - } - - #[must_use] - pub fn opener_count(&self) -> usize { - self.roles - .iter() - .filter(|role| role.opened_section().is_some()) - .count() - } - - fn opened_section(&self) -> Option { - self.roles.iter().find_map(|role| role.opened_section()) - } - - #[must_use] - pub fn span_section(&self, current: SampledTokenSection) -> SampledTokenSection { - self.opened_section().unwrap_or_else(|| { - if self - .roles - .iter() - .any(|role| role.closed_section() == Some(current)) - { - current - } else { - SampledTokenSection::Content - } - }) - } - - #[must_use] - pub fn next_section(&self) -> SampledTokenSection { - self.opened_section() - .unwrap_or(SampledTokenSection::Content) - } -} - -#[cfg(test)] -mod tests { - use super::StreamingMarker; - use crate::marker_role::MarkerRole; - use crate::token::LlamaToken; - - #[test] - fn a_role_the_marker_already_carries_is_not_added_twice() { - let mut marker = StreamingMarker::new(vec![LlamaToken::new(1)], MarkerRole::ReasoningClose); - - marker.add_role(MarkerRole::ToolCallOpen); - marker.add_role(MarkerRole::ToolCallOpen); - - assert_eq!( - marker.roles(), - &[MarkerRole::ReasoningClose, MarkerRole::ToolCallOpen] - ); - } -} diff --git a/llama-cpp-bindings/src/streaming_markers.rs b/llama-cpp-bindings/src/streaming_markers.rs deleted file mode 100644 index 8b9ed5e13..000000000 --- a/llama-cpp-bindings/src/streaming_markers.rs +++ /dev/null @@ -1,234 +0,0 @@ -use crate::error::MarkerDetectionError; -use crate::marker_role_candidate::MarkerRoleCandidate; -use crate::streaming_marker::StreamingMarker; -use crate::token::LlamaToken; - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -/// The normalized streaming markers detected for a model. -pub struct StreamingMarkers { - markers: Vec, -} - -impl StreamingMarkers { - /// # Errors - /// - /// Returns [`MarkerDetectionError::EmptyMarker`] when a candidate carries no tokens, - /// and [`MarkerDetectionError::AmbiguousMarkerOpeners`] when one token sequence would - /// open more than one section. - pub fn from_candidates( - candidates: impl IntoIterator, - ) -> Result { - let mut markers: Vec = Vec::new(); - - for MarkerRoleCandidate { tokens, role } in candidates { - if tokens.is_empty() { - return Err(MarkerDetectionError::EmptyMarker); - } - - if let Some(marker) = markers.iter_mut().find(|marker| marker.tokens() == tokens) { - marker.add_role(role); - } else { - markers.push(StreamingMarker::new(tokens, role)); - } - } - - if let Some(marker) = markers.iter().find(|marker| marker.opener_count() > 1) { - return Err(MarkerDetectionError::AmbiguousMarkerOpeners { - tokens: marker.tokens().to_vec(), - }); - } - - Ok(Self { markers }) - } - - #[must_use] - /// Returns whether the model exposes no streaming markers. - pub const fn is_empty(&self) -> bool { - self.markers.is_empty() - } - - #[must_use] - /// Returns the number of distinct marker token sequences. - pub const fn len(&self) -> usize { - self.markers.len() - } - - /// Iterates over the distinct marker token sequences. - pub fn iter(&self) -> impl Iterator { - self.markers.iter() - } - - #[must_use] - pub fn max_token_len(&self) -> usize { - self.markers - .iter() - .map(|marker| marker.tokens().len()) - .max() - .unwrap_or(0) - } - - #[must_use] - pub fn longest_matching_suffix( - &self, - reversed_tokens: &TReversedTokens, - ) -> Option<&StreamingMarker> - where - TReversedTokens: Iterator + Clone, - { - self.markers - .iter() - .filter(|marker| { - marker - .tokens() - .iter() - .rev() - .copied() - .eq(reversed_tokens.clone().take(marker.tokens().len())) - }) - .max_by_key(|marker| marker.tokens().len()) - } - - #[must_use] - pub fn is_prefix_of_longer_marker(&self, tokens: &[LlamaToken]) -> bool { - self.markers.iter().any(|marker| { - marker.tokens().len() > tokens.len() && marker.tokens().starts_with(tokens) - }) - } -} - -#[cfg(test)] -mod tests { - use super::StreamingMarkers; - use crate::error::MarkerDetectionError; - use crate::marker_role::MarkerRole; - use crate::marker_role_candidate::MarkerRoleCandidate; - use crate::token::LlamaToken; - - fn token(id: i32) -> LlamaToken { - LlamaToken::new(id) - } - - #[test] - fn empty_collection_reports_no_markers() { - let markers = StreamingMarkers::default(); - - assert!(markers.is_empty()); - assert_eq!(markers.len(), 0); - assert_eq!(markers.max_token_len(), 0); - } - - #[test] - fn candidates_with_the_same_tokens_are_one_marker_with_multiple_roles() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(1)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(1)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("a close and an opener compose into one transition"); - - let marker = markers.iter().next().expect("one marker must remain"); - assert_eq!(marker.tokens(), &[token(1)]); - assert_eq!( - marker.roles(), - &[MarkerRole::ReasoningClose, MarkerRole::ToolCallOpen] - ); - } - - #[test] - fn empty_marker_is_rejected() { - assert_eq!( - StreamingMarkers::from_candidates([MarkerRoleCandidate { - tokens: Vec::new(), - role: MarkerRole::ReasoningOpen - }]), - Err(MarkerDetectionError::EmptyMarker) - ); - } - - #[test] - fn two_openers_for_the_same_tokens_are_rejected() { - let marker_tokens = vec![token(1), token(2)]; - - assert_eq!( - StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: marker_tokens.clone(), - role: MarkerRole::ReasoningOpen - }, - MarkerRoleCandidate { - tokens: marker_tokens.clone(), - role: MarkerRole::ToolCallOpen - }, - ]), - Err(MarkerDetectionError::AmbiguousMarkerOpeners { - tokens: marker_tokens - }) - ); - } - - #[test] - fn longest_matching_suffix_wins() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(2)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(1), token(2)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("markers are valid"); - - let matched = markers - .longest_matching_suffix(&[token(1), token(2)].into_iter().rev()) - .expect("a suffix must match"); - - assert_eq!(matched.tokens(), &[token(1), token(2)]); - assert_eq!(matched.roles(), &[MarkerRole::ToolCallOpen]); - } - - #[test] - fn shorter_complete_marker_reports_when_it_is_still_an_ambiguous_prefix() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(1)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(1), token(2)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("markers are valid"); - - assert!(markers.is_prefix_of_longer_marker(&[token(1)])); - assert!(!markers.is_prefix_of_longer_marker(&[token(1), token(2)])); - } - - #[test] - fn max_token_len_uses_the_longest_normalized_marker() { - let markers = StreamingMarkers::from_candidates([ - MarkerRoleCandidate { - tokens: vec![token(1)], - role: MarkerRole::ReasoningOpen, - }, - MarkerRoleCandidate { - tokens: vec![token(2), token(3), token(4)], - role: MarkerRole::ReasoningClose, - }, - MarkerRoleCandidate { - tokens: vec![token(5), token(6)], - role: MarkerRole::ToolCallOpen, - }, - ]) - .expect("markers are valid"); - - assert_eq!(markers.max_token_len(), 3); - } -} diff --git a/llama-cpp-bindings/src/synthetic_tool_call_renders.rs b/llama-cpp-bindings/src/synthetic_tool_call_renders.rs deleted file mode 100644 index 2a14fa1da..000000000 --- a/llama-cpp-bindings/src/synthetic_tool_call_renders.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct SyntheticToolCallRenders { - pub without_tools: String, - pub with_tools: String, -} diff --git a/llama-cpp-bindings/src/timing.rs b/llama-cpp-bindings/src/timing.rs deleted file mode 100644 index ed68ec2fe..000000000 --- a/llama-cpp-bindings/src/timing.rs +++ /dev/null @@ -1,282 +0,0 @@ -use std::fmt::{Debug, Display, Formatter}; -fn write_timings(timings: &LlamaTimings, writer: &mut dyn std::fmt::Write) -> std::fmt::Result { - writeln!(writer, "load time = {:.2} ms", timings.t_load_ms())?; - - if timings.n_p_eval() > 0 { - writeln!( - writer, - "prompt eval time = {:.2} ms / {} tokens ({:.2} ms per token, {:.2} tokens per second)", - timings.t_p_eval_ms(), - timings.n_p_eval(), - timings.t_p_eval_ms() / f64::from(timings.n_p_eval()), - 1e3 / timings.t_p_eval_ms() * f64::from(timings.n_p_eval()) - )?; - } else { - writeln!( - writer, - "prompt eval time = {:.2} ms / 0 tokens", - timings.t_p_eval_ms(), - )?; - } - - if timings.n_eval() > 0 { - writeln!( - writer, - "eval time = {:.2} ms / {} runs ({:.2} ms per token, {:.2} tokens per second)", - timings.t_eval_ms(), - timings.n_eval(), - timings.t_eval_ms() / f64::from(timings.n_eval()), - 1e3 / timings.t_eval_ms() * f64::from(timings.n_eval()) - )?; - } else { - writeln!(writer, "eval time = {:.2} ms / 0 runs", timings.t_eval_ms())?; - } - - Ok(()) -} - -#[derive(Clone, Copy, Debug)] -pub struct LlamaTimings { - pub timings: llama_cpp_bindings_sys::llama_perf_context_data, -} - -impl LlamaTimings { - #[must_use] - pub const fn new( - t_start_ms: f64, - t_load_ms: f64, - t_p_eval_ms: f64, - t_eval_ms: f64, - n_p_eval: i32, - n_eval: i32, - n_reused: i32, - ) -> Self { - Self { - timings: llama_cpp_bindings_sys::llama_perf_context_data { - t_start_ms, - t_load_ms, - t_p_eval_ms, - t_eval_ms, - n_p_eval, - n_eval, - n_reused, - }, - } - } - - #[must_use] - pub const fn t_start_ms(&self) -> f64 { - self.timings.t_start_ms - } - - #[must_use] - pub const fn t_load_ms(&self) -> f64 { - self.timings.t_load_ms - } - - #[must_use] - pub const fn t_p_eval_ms(&self) -> f64 { - self.timings.t_p_eval_ms - } - - #[must_use] - pub const fn t_eval_ms(&self) -> f64 { - self.timings.t_eval_ms - } - - #[must_use] - pub const fn n_p_eval(&self) -> i32 { - self.timings.n_p_eval - } - - #[must_use] - pub const fn n_eval(&self) -> i32 { - self.timings.n_eval - } - - pub const fn set_t_start_ms(&mut self, t_start_ms: f64) { - self.timings.t_start_ms = t_start_ms; - } - - pub const fn set_t_load_ms(&mut self, t_load_ms: f64) { - self.timings.t_load_ms = t_load_ms; - } - - pub const fn set_t_p_eval_ms(&mut self, t_p_eval_ms: f64) { - self.timings.t_p_eval_ms = t_p_eval_ms; - } - - pub const fn set_t_eval_ms(&mut self, t_eval_ms: f64) { - self.timings.t_eval_ms = t_eval_ms; - } - - pub const fn set_n_p_eval(&mut self, n_p_eval: i32) { - self.timings.n_p_eval = n_p_eval; - } - - pub const fn set_n_eval(&mut self, n_eval: i32) { - self.timings.n_eval = n_eval; - } -} - -impl Display for LlamaTimings { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { - write_timings(self, formatter) - } -} - -#[cfg(test)] -mod tests { - use super::LlamaTimings; - - #[test] - fn display_format_with_valid_counts() { - let timings = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 5, 6, 1); - let output = format!("{timings}"); - - assert!(output.contains("load time = 2.00 ms")); - assert!(output.contains("prompt eval time = 3.00 ms / 5 tokens")); - assert!(output.contains("eval time = 4.00 ms / 6 runs")); - } - - #[test] - fn display_format_handles_zero_eval_counts() { - let timings = LlamaTimings::new(0.0, 1.0, 2.0, 3.0, 0, 0, 0); - let output = format!("{timings}"); - - assert!(output.contains("load time = 1.00 ms")); - assert!(output.contains("prompt eval time = 2.00 ms / 0 tokens")); - assert!(output.contains("eval time = 3.00 ms / 0 runs")); - assert!(!output.contains("NaN")); - assert!(!output.contains("inf")); - } - - #[test] - fn set_t_start_ms() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_t_start_ms(42.0); - - assert!((timings.t_start_ms() - 42.0).abs() < f64::EPSILON); - } - - #[test] - fn set_t_load_ms() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_t_load_ms(10.5); - - assert!((timings.t_load_ms() - 10.5).abs() < f64::EPSILON); - } - - #[test] - fn set_t_p_eval_ms() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_t_p_eval_ms(7.7); - - assert!((timings.t_p_eval_ms() - 7.7).abs() < f64::EPSILON); - } - - #[test] - fn set_t_eval_ms() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_t_eval_ms(3.3); - - assert!((timings.t_eval_ms() - 3.3).abs() < f64::EPSILON); - } - - #[test] - fn set_n_p_eval() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_n_p_eval(100); - - assert_eq!(timings.n_p_eval(), 100); - } - - #[test] - fn set_n_eval() { - let mut timings = LlamaTimings::new(0.0, 0.0, 0.0, 0.0, 0, 0, 0); - - timings.set_n_eval(200); - - assert_eq!(timings.n_eval(), 200); - } - - #[test] - fn write_timings_propagates_writer_errors() { - struct FailingWriter; - - impl std::fmt::Write for FailingWriter { - fn write_str(&mut self, _text: &str) -> std::fmt::Result { - Err(std::fmt::Error) - } - } - - let timings = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 5, 6, 1); - let result = super::write_timings(&timings, &mut FailingWriter); - - assert!(result.is_err()); - } - - #[test] - fn write_timings_zero_p_eval_with_failing_writer() { - struct FailAfterNWrites { - remaining: usize, - } - - impl std::fmt::Write for FailAfterNWrites { - fn write_str(&mut self, _text: &str) -> std::fmt::Result { - if self.remaining == 0 { - return Err(std::fmt::Error); - } - self.remaining -= 1; - - Ok(()) - } - } - - let timings = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 0, 6, 1); - let result = super::write_timings(&timings, &mut FailAfterNWrites { remaining: 1 }); - - assert!(result.is_err()); - } - - #[test] - fn write_timings_fails_at_each_writeln_boundary() { - struct FailAfterNWrites { - remaining: usize, - } - - impl std::fmt::Write for FailAfterNWrites { - fn write_str(&mut self, _text: &str) -> std::fmt::Result { - if self.remaining == 0 { - return Err(std::fmt::Error); - } - self.remaining -= 1; - - Ok(()) - } - } - - let with_counts = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 5, 6, 1); - let zero_counts = LlamaTimings::new(1.0, 2.0, 3.0, 4.0, 0, 0, 1); - - for writes_allowed in 0..20 { - let _ = super::write_timings( - &with_counts, - &mut FailAfterNWrites { - remaining: writes_allowed, - }, - ); - let _ = super::write_timings( - &zero_counts, - &mut FailAfterNWrites { - remaining: writes_allowed, - }, - ); - } - } -} diff --git a/llama-cpp-bindings/src/token.rs b/llama-cpp-bindings/src/token.rs index 4b87459ee..8af2c903d 100644 --- a/llama-cpp-bindings/src/token.rs +++ b/llama-cpp-bindings/src/token.rs @@ -1,5 +1,7 @@ +use std::fmt; use std::fmt::Debug; use std::fmt::Display; +use std::fmt::Formatter; pub mod data; pub mod data_array; @@ -10,7 +12,7 @@ pub mod logit_bias; pub struct LlamaToken(pub llama_cpp_bindings_sys::llama_token); impl Display for LlamaToken { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) } } diff --git a/llama-cpp-bindings/src/token/data_array.rs b/llama-cpp-bindings/src/token/data_array.rs index 497121898..3943e5402 100644 --- a/llama-cpp-bindings/src/token/data_array.rs +++ b/llama-cpp-bindings/src/token/data_array.rs @@ -1,59 +1,9 @@ use std::ptr; -use crate::error::SamplerApplyError; -use crate::error::TokenSamplingError; -use crate::sampling::LlamaSampler; -use crate::token::data::LlamaTokenData; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; use super::LlamaToken; -use llama_cpp_ffi_status::read_and_free_cpp_string; - -fn sampler_apply_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_sampler_apply_status, - out_error: *mut std::os::raw::c_char, -) -> Result<(), SamplerApplyError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_OK => Ok(()), - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_SAMPLER_ARG => { - Err(SamplerApplyError::NullSampler) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED => { - Err(SamplerApplyError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_OUT_OF_MEMORY => { - Err(SamplerApplyError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_sampler_apply", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(SamplerApplyError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "was given a null data_array argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "was given a null out_error argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_sampler_apply", - code: i64::from(other), - } - .into()), - } -} +use crate::token::data::LlamaTokenData; #[derive(Debug, Clone, PartialEq)] pub struct LlamaTokenDataArray { @@ -72,13 +22,6 @@ impl LlamaTokenDataArray { } } - pub fn from_iter(data: TIterator, sorted: bool) -> Self - where - TIterator: IntoIterator, - { - Self::new(data.into_iter().collect(), sorted) - } - pub fn replace_candidates(&mut self, candidates: TIterator) where TIterator: IntoIterator, @@ -93,12 +36,49 @@ impl LlamaTokenDataArray { pub fn selected_token(&self) -> Option { self.data.get(self.selected?).map(LlamaTokenData::id) } + + /// Whether a sampler left at least one candidate it could still select. + #[must_use] + pub fn has_selectable_candidate(&self) -> bool { + self.data + .iter() + .any(|candidate| candidate.logit() > f32::NEG_INFINITY) + } } impl LlamaTokenDataArray { + fn c_selection(&self) -> Result { + let Some(selected_index) = self.selected else { + return Ok(-1); + }; + + self.data + .get(selected_index) + .and_then(|_selected_candidate| i64::try_from(selected_index).ok()) + .ok_or(FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the token data array selects a token outside itself", + }) + } + + fn selection_of(&self, c_selected: i64) -> Result, FfiContractError> { + if c_selected == -1 { + return Ok(None); + } + + usize::try_from(c_selected) + .ok() + .filter(|&selected_index| selected_index < self.data.len()) + .map(Some) + .ok_or(FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the llama.cpp sampler selected a token outside the token data array", + }) + } + /// # Errors /// - /// Returns [`crate::FfiContractError`] when the llama.cpp sampler grows the array beyond the + /// Returns [`llama_cpp_ffi_status::ffi_contract_error::FfiContractError`] when the llama.cpp sampler grows the array beyond the /// capacity this buffer was allocated with, which would make the following `set_len` /// undefined behaviour. /// @@ -110,7 +90,7 @@ impl LlamaTokenDataArray { pub unsafe fn modify_as_c_llama_token_data_array( &mut self, modify: impl FnOnce(&mut llama_cpp_bindings_sys::llama_token_data_array) -> TResult, - ) -> Result { + ) -> Result { let size = self.data.len(); let data = self .data @@ -120,17 +100,14 @@ impl LlamaTokenDataArray { let mut c_llama_token_data_array = llama_cpp_bindings_sys::llama_token_data_array { data, size, - selected: self - .selected - .and_then(|selected_index| selected_index.try_into().ok()) - .unwrap_or(-1), + selected: self.c_selection()?, sorted: self.sorted, }; let result = modify(&mut c_llama_token_data_array); if c_llama_token_data_array.size > self.data.capacity() { - return Err(crate::FfiContractError { + return Err(FfiContractError { operation: "modify_as_c_llama_token_data_array", detail: "the llama.cpp sampler grew the token data array beyond its capacity", }); @@ -148,199 +125,20 @@ impl LlamaTokenDataArray { } self.sorted = c_llama_token_data_array.sorted; - self.selected = c_llama_token_data_array - .selected - .try_into() - .ok() - .filter(|&selected_index| selected_index < self.data.len()); + self.selected = self.selection_of(c_llama_token_data_array.selected)?; Ok(result) } - - /// # Errors - /// - /// Returns [`SamplerApplyError`] if the sampler pointer is null, the llama.cpp - /// sampler runs out of memory, it throws a C++ exception while applying, or it - /// excludes every candidate so that no token is left to select. - pub fn apply_sampler(&mut self, sampler: &LlamaSampler) -> Result<(), SamplerApplyError> { - let applied = unsafe { - self.modify_as_c_llama_token_data_array(|c_llama_token_data_array| { - let mut out_error: *mut std::os::raw::c_char = ptr::null_mut(); - let status = llama_cpp_bindings_sys::llama_rs_sampler_apply( - sampler.as_ptr(), - c_llama_token_data_array, - &raw mut out_error, - ); - sampler_apply_status_to_result(status, out_error) - })? - }; - - applied.and_then(|()| self.ensure_a_candidate_remains()) - } - - fn ensure_a_candidate_remains(&self) -> Result<(), SamplerApplyError> { - if self - .data - .iter() - .any(|candidate| candidate.logit() > f32::NEG_INFINITY) - { - Ok(()) - } else { - Err(SamplerApplyError::EveryCandidateExcluded) - } - } - - /// # Errors - /// Returns [`SamplerApplyError`] if applying the sampler fails. - pub fn with_sampler(mut self, sampler: &mut LlamaSampler) -> Result { - self.apply_sampler(sampler)?; - Ok(self) - } - - /// # Errors - /// Returns [`TokenSamplingError::SamplerApply`] if applying the sampler fails, or - /// [`TokenSamplingError::NoTokenSelected`] if the sampler fails to select a token. - pub fn sample_token(&mut self, seed: u32) -> Result { - let sampler = LlamaSampler::dist(seed)?; - self.apply_sampler(&sampler)?; - self.selected_token() - .ok_or(TokenSamplingError::NoTokenSelected) - } - - /// # Errors - /// Returns [`TokenSamplingError::SamplerApply`] if applying the sampler fails, or - /// [`TokenSamplingError::NoTokenSelected`] if the sampler fails to select a token. - pub fn sample_token_greedy(&mut self) -> Result { - let sampler = LlamaSampler::greedy()?; - self.apply_sampler(&sampler)?; - self.selected_token() - .ok_or(TokenSamplingError::NoTokenSelected) - } } #[cfg(test)] mod tests { + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use super::LlamaTokenDataArray; - use crate::error::SamplerApplyError; use crate::token::LlamaToken; use crate::token::data::LlamaTokenData; - #[test] - fn a_sampler_excluding_every_candidate_leaves_nothing_to_select() { - use crate::sampling::LlamaSampler; - use crate::token::logit_bias::LlamaLogitBias; - - let mut array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(1), 2.0, 0.0), - ], - false, - ); - let excluding_sampler = LlamaSampler::logit_bias( - 2, - &[ - LlamaLogitBias::new(LlamaToken::new(0), f32::NEG_INFINITY), - LlamaLogitBias::new(LlamaToken::new(1), f32::NEG_INFINITY), - ], - ) - .expect("the logit bias sampler must initialize"); - - assert_eq!( - array.apply_sampler(&excluding_sampler), - Err(SamplerApplyError::EveryCandidateExcluded) - ); - } - - #[test] - fn apply_greedy_sampler_selects_highest_logit() { - use crate::sampling::LlamaSampler; - - let mut array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(1), 5.0, 0.0), - LlamaTokenData::new(LlamaToken::new(2), 3.0, 0.0), - ], - false, - ); - - let sampler = LlamaSampler::greedy().expect("greedy sampler must initialize"); - array - .apply_sampler(&sampler) - .expect("test: greedy sampler must apply"); - - assert_eq!(array.selected_token(), Some(LlamaToken::new(1))); - } - - #[test] - fn with_sampler_builder_pattern() { - use crate::sampling::LlamaSampler; - - let mut sampler = LlamaSampler::greedy().expect("greedy sampler must initialize"); - let array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(1), 5.0, 0.0), - ], - false, - ) - .with_sampler(&mut sampler) - .expect("test: building with greedy sampler must succeed"); - - assert_eq!(array.selected_token(), Some(LlamaToken::new(1))); - } - - #[test] - fn sample_token_greedy_returns_highest() { - let mut array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(10), 0.1, 0.0), - LlamaTokenData::new(LlamaToken::new(20), 9.9, 0.0), - ], - false, - ); - - let token = array - .sample_token_greedy() - .expect("test: greedy sampler should select a token"); - - assert_eq!(token, LlamaToken::new(20)); - } - - #[test] - fn from_iter_creates_array_from_iterator() { - let array = LlamaTokenDataArray::from_iter( - [ - LlamaTokenData::new(LlamaToken::new(0), 0.0, 0.0), - LlamaTokenData::new(LlamaToken::new(1), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(2), 2.0, 0.0), - ], - false, - ); - - assert_eq!(array.data.len(), 3); - assert!(!array.sorted); - assert!(array.selected.is_none()); - } - - #[test] - fn sample_token_with_seed_selects_a_token() { - let mut array = LlamaTokenDataArray::new( - vec![ - LlamaTokenData::new(LlamaToken::new(10), 1.0, 0.0), - LlamaTokenData::new(LlamaToken::new(20), 1.0, 0.0), - ], - false, - ); - - let token = array - .sample_token(42) - .expect("test: dist sampler should select a token"); - - assert!(token == LlamaToken::new(10) || token == LlamaToken::new(20)); - } - #[test] fn selected_token_returns_none_when_no_selection() { let array = LlamaTokenDataArray::new( @@ -348,7 +146,7 @@ mod tests { false, ); - assert!(array.selected_token().is_none()); + assert_eq!(array.selected_token(), None); } #[test] @@ -359,7 +157,7 @@ mod tests { sorted: false, }; - assert!(array.selected_token().is_none()); + assert_eq!(array.selected_token(), None); } #[test] @@ -403,37 +201,45 @@ mod tests { } #[test] - fn modify_clears_selection_when_index_is_out_of_range() { + fn a_sampler_selecting_a_token_outside_the_array_is_a_contract_violation() { let mut array = LlamaTokenDataArray::new( vec![LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0)], false, ); - unsafe { - array - .modify_as_c_llama_token_data_array(|c_array| { - c_array.selected = 5; - }) - .expect("the array is left at its original size"); - } + let result = unsafe { + array.modify_as_c_llama_token_data_array(|c_array| { + c_array.selected = 5; + }) + }; - assert_eq!(array.selected, None); + assert_eq!( + result, + Err(FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the llama.cpp sampler selected a token outside the token data array", + }) + ); } #[test] - fn selected_overflow_uses_negative_one() { - let mut array = LlamaTokenDataArray { - data: vec![LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0)], - selected: Some(usize::MAX), - sorted: false, - }; - - unsafe { - array - .modify_as_c_llama_token_data_array(|c_array| { - assert_eq!(c_array.selected, -1); + fn a_selection_outside_the_array_is_refused_before_reaching_llama_cpp() { + for selected in [1, usize::MAX] { + let mut array = LlamaTokenDataArray { + data: vec![LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0)], + selected: Some(selected), + sorted: false, + }; + + let result = unsafe { array.modify_as_c_llama_token_data_array(|_c_array| ()) }; + + assert_eq!( + result, + Err(FfiContractError { + operation: "modify_as_c_llama_token_data_array", + detail: "the token data array selects a token outside itself", }) - .expect("the array is left at its original size"); + ); } } @@ -453,7 +259,7 @@ mod tests { assert_eq!( result, - Err(crate::FfiContractError { + Err(FfiContractError { operation: "modify_as_c_llama_token_data_array", detail: "the llama.cpp sampler grew the token data array beyond its capacity", }) @@ -483,104 +289,3 @@ mod tests { assert_eq!(array.selected, Some(1)); } } - -#[cfg(test)] -mod sampler_apply_status_to_result_tests { - use std::ptr; - - use super::sampler_apply_status_to_result; - use crate::error::SamplerApplyError; - - #[test] - fn sampler_apply_status_allocation_failed_returns_not_enough_memory() { - assert_eq!( - sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_ERROR_STRING_ALLOCATION_FAILED, - ptr::null_mut(), - ), - Err(SamplerApplyError::NotEnoughMemory), - ); - } - - #[test] - fn sampler_apply_status_cxx_exception_without_a_message_is_a_contract_error() { - assert_eq!( - sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_THREW_CXX_EXCEPTION, - ptr::null_mut(), - ), - Err(crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "reported a thrown C++ exception without an error message", - } - .into()), - ); - } - - #[test] - fn sampler_apply_unknown_status_is_preserved() { - let result = sampler_apply_status_to_result(255, ptr::null_mut()); - - assert_eq!( - result, - Err(SamplerApplyError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_sampler_apply", - code: 255, - })) - ); - } - - #[test] - fn sampler_apply_status_to_result_maps_null_sampler_arg() { - let outcome = sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_SAMPLER_ARG, - ptr::null_mut(), - ); - assert_eq!(outcome.err(), Some(SamplerApplyError::NullSampler)); - } - - #[test] - fn sampler_apply_status_to_result_maps_null_data_array_arg() { - let outcome = sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_DATA_ARRAY_ARG, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "was given a null data_array argument", - } - .into() - ) - ); - } - - #[test] - fn sampler_apply_status_to_result_maps_null_out_error_arg() { - let outcome = sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_NULL_OUT_ERROR_ARG, - ptr::null_mut(), - ); - assert_eq!( - outcome.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_sampler_apply", - detail: "was given a null out_error argument", - } - .into() - ) - ); - } - - #[test] - fn sampler_apply_status_to_result_maps_llama_cpp_out_of_memory() { - let outcome = sampler_apply_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_SAMPLER_APPLY_LLAMA_CPP_OUT_OF_MEMORY, - ptr::null_mut(), - ); - assert_eq!(outcome.err(), Some(SamplerApplyError::LlamaCppOutOfMemory)); - } -} diff --git a/llama-cpp-bindings/src/token_attribution.rs b/llama-cpp-bindings/src/token_attribution.rs new file mode 100644 index 000000000..c7bffd118 --- /dev/null +++ b/llama-cpp-bindings/src/token_attribution.rs @@ -0,0 +1,6 @@ +use crate::chat_session_attribution::ChatSessionAttribution; + +pub enum TokenAttribution<'session, 'model> { + Undeterminable, + ChatSession(ChatSessionAttribution<'session, 'model>), +} diff --git a/llama-cpp-bindings/src/token_piece.rs b/llama-cpp-bindings/src/token_piece.rs index 5bef2b216..d7ca5fd1b 100644 --- a/llama-cpp-bindings/src/token_piece.rs +++ b/llama-cpp-bindings/src/token_piece.rs @@ -23,7 +23,7 @@ impl TokenPiece { #[cfg(test)] mod tests { - use super::TokenPiece; + use crate::token_piece::TokenPiece; #[test] fn a_marker_is_raw_text_without_a_visible_part() { diff --git a/llama-cpp-bindings/src/tool_call_format.rs b/llama-cpp-bindings/src/tool_call_format.rs deleted file mode 100644 index 131032a7a..000000000 --- a/llama-cpp-bindings/src/tool_call_format.rs +++ /dev/null @@ -1,404 +0,0 @@ -pub mod bracketed_args; -pub mod consume_optional_prefix; -pub mod json_object; -pub mod key_value_xml_tags; -pub mod paired_quote_args; -pub mod parse_step; -pub mod scalar_value_to_json; -pub mod separator_split; -pub mod tool_call_format_outcome; -pub mod xml_function_tags; - -pub use self::tool_call_format_outcome::ToolCallFormatOutcome; - -use llama_cpp_bindings_types::ToolCallArgsShape; -use llama_cpp_bindings_types::ToolCallMarkers; - -use crate::error::ToolCallFormatFailure; - -#[must_use] -pub fn try_parse(body: &str, markers: &ToolCallMarkers) -> ToolCallFormatOutcome { - if markers.open.is_empty() { - return ToolCallFormatOutcome::NoMatch; - } - - let parsed: Result, ToolCallFormatFailure> = match &markers.args_shape { - ToolCallArgsShape::BracketedJson(shape) => { - bracketed_args::parse(body, markers, shape).map_err(Into::into) - } - ToolCallArgsShape::JsonObject(shape) => json_object::parse(body, shape).map_err(Into::into), - ToolCallArgsShape::KeyValueXmlTags(shape) => { - key_value_xml_tags::parse(body, markers, shape).map_err(Into::into) - } - ToolCallArgsShape::PairedQuote(shape) => { - paired_quote_args::parse(body, markers, shape).map_err(Into::into) - } - ToolCallArgsShape::XmlTags(shape) => { - xml_function_tags::parse(body, shape).map_err(Into::into) - } - }; - - match parsed { - Ok(parsed) if parsed.is_empty() => ToolCallFormatOutcome::NoMatch, - Ok(parsed) => ToolCallFormatOutcome::Parsed(parsed), - Err(failure) => ToolCallFormatOutcome::Failed(failure), - } -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::BracketedJsonShape; - use llama_cpp_bindings_types::KeyValueXmlTagsShape; - use llama_cpp_bindings_types::PairedQuoteShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ParsedToolCallId; - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::ToolCallMarkers; - use llama_cpp_bindings_types::ToolCallValueQuote; - use llama_cpp_bindings_types::XmlTagsShape; - use serde_json::json; - - use super::ToolCallFormatOutcome; - use super::try_parse; - use crate::error::BracketedArgsFailure; - use crate::error::ToolCallFormatFailure; - - fn mistral3_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "[TOOL_CALLS]".to_owned(), - close: String::new(), - args_shape: ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - } - } - - fn gemma4_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "<|tool_call>call:".to_owned(), - close: "}".to_owned(), - args_shape: ToolCallArgsShape::PairedQuote(PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }), - } - } - - fn qwen35_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::XmlTags(XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - }), - } - } - - fn glm47_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::KeyValueXmlTags(KeyValueXmlTagsShape { - key_open: "".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - }), - } - } - - #[test] - fn dispatches_to_bracketed_args_for_mistral3_shape() { - let outcome = try_parse( - "[TOOL_CALLS]get_weather[ARGS]{\"location\":\"Paris\"}", - &mistral3_markers(), - ); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - )]), - ); - } - - #[test] - fn dispatches_to_paired_quote_args_for_gemma4_shape() { - let outcome = try_parse( - "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}", - &gemma4_markers(), - ); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - )]), - ); - } - - #[test] - fn dispatches_to_key_value_xml_tags_for_glm47_shape() { - let outcome = try_parse( - "get_weatherlocationParis", - &glm47_markers(), - ); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - )]), - ); - } - - #[test] - fn dispatches_to_xml_function_tags_for_qwen35_shape() { - let outcome = try_parse( - "Paris", - &qwen35_markers(), - ); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Parsed(vec![ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - "get_weather".to_owned(), - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - )]), - ); - } - - #[test] - fn no_match_when_open_marker_is_empty() { - let markers = ToolCallMarkers { - open: String::new(), - close: String::new(), - args_shape: ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - }; - - assert_eq!( - try_parse("[TOOL_CALLS]get_weather[ARGS]{}", &markers), - ToolCallFormatOutcome::NoMatch, - ); - } - - #[test] - fn no_match_when_body_lacks_markers() { - assert_eq!( - try_parse("plain text without tool calls", &mistral3_markers()), - ToolCallFormatOutcome::NoMatch, - ); - } - - #[test] - fn failed_when_inner_parser_returns_typed_failure() { - let outcome = try_parse("[TOOL_CALLS]get_weather[ARGS] ", &mistral3_markers()); - - assert_eq!( - outcome, - ToolCallFormatOutcome::Failed(ToolCallFormatFailure::BracketedArgs( - BracketedArgsFailure::UnterminatedArguments { - tool_name: "get_weather".to_owned(), - }, - )), - ); - } - - #[test] - fn try_parse_returns_no_match_for_glm_input_under_qwen_markers() { - let glm_input = "get_weather\ - location\ - Paris\ - "; - - assert_eq!( - try_parse(glm_input, &qwen35_markers()), - ToolCallFormatOutcome::NoMatch, - ); - } - - #[test] - fn try_parse_returns_no_match_for_plain_content_under_every_known_shape() { - use crate::chat_template_tool_calls::known_marker_candidates; - - let plain_content = "Sorry, I cannot help with that request."; - - for candidate in known_marker_candidates() { - assert_eq!( - try_parse(plain_content, &candidate), - ToolCallFormatOutcome::NoMatch, - "expected NoMatch for plain content under candidate {candidate:?}" - ); - } - } - - #[test] - fn duck_type_resolves_qwen_xml_input_via_xml_tags_shape_first() { - use llama_cpp_bindings_types::ToolCallArguments; - - use crate::chat_template_tool_calls::known_marker_candidates; - - let qwen_input = "\n\ - \n\ - \n\ - Paris\n\ - \n\ - \n\ - "; - - let mut resolved = None; - for candidate in known_marker_candidates() { - if let ToolCallFormatOutcome::Parsed(calls) = try_parse(qwen_input, &candidate) { - resolved = Some((candidate.args_shape, calls)); - break; - } - } - - let (args_shape, calls) = - resolved.expect("Qwen XML input must resolve via at least one duck-type candidate"); - assert_eq!( - args_shape, - ToolCallArgsShape::XmlTags(XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - }), - "duck-type ordering must resolve Qwen XML via the XmlTags shape (most restrictive \ - shape that requires `".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - }), - "GLM input must resolve via the KeyValueXmlTags shape, got {args_shape:?}" - ); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "get_weather"); - assert_eq!( - calls[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn duck_type_resolves_mistral_input_via_bracketed_json_shape() { - use llama_cpp_bindings_types::ToolCallArguments; - - use crate::chat_template_tool_calls::known_marker_candidates; - - let mistral_input = r#"[TOOL_CALLS]get_weather[ARGS]{"location":"Paris"}"#; - - let mut resolved = None; - for candidate in known_marker_candidates() { - if let ToolCallFormatOutcome::Parsed(calls) = try_parse(mistral_input, &candidate) { - resolved = Some((candidate.args_shape, calls)); - break; - } - } - - let (args_shape, calls) = - resolved.expect("Mistral input must resolve via at least one duck-type candidate"); - assert_eq!( - args_shape, - ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - "Mistral input must resolve via the BracketedJson shape; the candidate ordering must \ - try BracketedJson before PairedQuote because PairedQuote's `{{` separator could \ - greedily match Mistral's JSON args. Got {args_shape:?}" - ); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "get_weather"); - assert_eq!( - calls[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn duck_type_resolves_gemma_input_via_paired_quote_shape() { - use llama_cpp_bindings_types::ToolCallArguments; - - use crate::chat_template_tool_calls::known_marker_candidates; - - let gemma_input = "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}"; - - let mut resolved = None; - for candidate in known_marker_candidates() { - if let ToolCallFormatOutcome::Parsed(calls) = try_parse(gemma_input, &candidate) { - resolved = Some((candidate.args_shape, calls)); - break; - } - } - - let (args_shape, calls) = - resolved.expect("Gemma input must resolve via at least one duck-type candidate"); - assert_eq!( - args_shape, - ToolCallArgsShape::PairedQuote(PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }), - "Gemma input must resolve via the PairedQuote shape, got {args_shape:?}" - ); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "get_weather"); - assert_eq!( - calls[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs b/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs deleted file mode 100644 index 3fb0e4367..000000000 --- a/llama-cpp-bindings/src/tool_call_format/bracketed_args.rs +++ /dev/null @@ -1,262 +0,0 @@ -use llama_cpp_bindings_types::BracketedJsonShape; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_bindings_types::ToolCallMarkers; - -use crate::error::BracketedArgsFailure; -use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; -use crate::tool_call_format::parse_step::ParseStep; -use crate::tool_call_format::separator_split::SeparatorSplit; - -fn consume_one_json_value<'body>( - input: &'body str, - tool_name: &str, -) -> Result<(serde_json::Value, &'body str), BracketedArgsFailure> { - let mut stream = serde_json::Deserializer::from_str(input).into_iter::(); - let value = stream - .next() - .ok_or_else(|| BracketedArgsFailure::UnterminatedArguments { - tool_name: tool_name.to_owned(), - })? - .map_err(|err| BracketedArgsFailure::InvalidJsonArguments { - tool_name: tool_name.to_owned(), - message: err.to_string(), - })?; - let consumed = stream.byte_offset(); - - Ok((value, &input[consumed..])) -} - -fn parse_one_call<'body>( - input: &'body str, - markers: &ToolCallMarkers, - shape: &BracketedJsonShape, -) -> Result, BracketedArgsFailure> { - if input.is_empty() { - return Ok(ParseStep::Done); - } - - let after_open = consume_optional_prefix(input, markers.open.as_str()); - - let Some(SeparatorSplit { - before: name_raw, - after: after_separator, - }) = SeparatorSplit::at_first(after_open, shape.name_args_separator.as_str()) - else { - return Ok(ParseStep::Done); - }; - - let name = name_raw.trim().to_owned(); - if name.is_empty() { - return Ok(ParseStep::Done); - } - - let (arguments_value, after_arguments) = consume_one_json_value(after_separator, &name)?; - - let after_close = consume_optional_prefix(after_arguments, markers.close.as_str()); - - Ok(ParseStep::Call { - call: ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - name, - ToolCallArguments::ValidJson(arguments_value), - ), - remainder: after_close, - }) -} - -/// # Errors -/// -/// Returns [`BracketedArgsFailure`] when the body looks like a bracketed-JSON -/// tool-call block (matches the name/args separator) but contains a structural -/// issue: invalid JSON arguments or a JSON value truncated mid-stream. -pub fn parse( - body: &str, - markers: &ToolCallMarkers, - shape: &BracketedJsonShape, -) -> Result, BracketedArgsFailure> { - if shape.name_args_separator.is_empty() { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body.trim_start(); - - loop { - match parse_one_call(remaining, markers, shape)? { - ParseStep::Done => break, - ParseStep::Call { call, remainder } => { - parsed.push(call); - remaining = remainder.trim_start(); - } - } - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::BracketedJsonShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::ToolCallMarkers; - use serde_json::json; - - use super::parse; - use crate::error::BracketedArgsFailure; - - fn mistral3_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "[TOOL_CALLS]".to_owned(), - close: String::new(), - args_shape: ToolCallArgsShape::BracketedJson(BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - }), - } - } - - fn mistral3_shape() -> BracketedJsonShape { - BracketedJsonShape { - name_args_separator: "[ARGS]".to_owned(), - } - } - - #[test] - fn parses_single_tool_call_with_open_marker_present() { - let parsed = parse( - "[TOOL_CALLS]get_weather[ARGS]{\"location\":\"Paris\"}", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_single_tool_call_when_classifier_stripped_open_marker() { - let parsed = parse( - "get_weather[ARGS]{\"location\":\"Paris\"}", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_two_consecutive_tool_calls_with_repeated_open_marker() { - let parsed = parse( - "[TOOL_CALLS]a[ARGS]{\"x\":1}[TOOL_CALLS]b[ARGS]{\"y\":2}", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 2); - assert_eq!(parsed[0].name, "a"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"x": 1})) - ); - assert_eq!(parsed[1].name, "b"); - assert_eq!( - parsed[1].arguments, - ToolCallArguments::ValidJson(json!({"y": 2})) - ); - } - - #[test] - fn rejects_malformed_json_arguments_with_typed_failure() { - let result = parse( - "[TOOL_CALLS]get_weather[ARGS]{\"location\":}", - &mistral3_markers(), - &mistral3_shape(), - ); - - let failure = result.expect_err("malformed JSON must produce a typed failure"); - - assert_eq!( - std::mem::discriminant(&failure), - std::mem::discriminant(&BracketedArgsFailure::InvalidJsonArguments { - tool_name: String::new(), - message: String::new(), - }), - ); - } - - #[test] - fn rejects_truncated_json_arguments_with_unterminated_failure() { - let failure = parse( - "[TOOL_CALLS]get_weather[ARGS] ", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect_err("truncated arguments must produce a typed failure"); - - assert_eq!( - failure, - BracketedArgsFailure::UnterminatedArguments { - tool_name: "get_weather".to_owned(), - }, - ); - } - - #[test] - fn returns_empty_vec_for_separator_with_only_whitespace_name() { - let parsed = parse( - "[TOOL_CALLS] [ARGS]{\"x\":1}", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("whitespace-name input must parse"); - - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_when_shape_has_empty_separator() { - let mut shape = mistral3_shape(); - shape.name_args_separator.clear(); - let parsed = parse( - "[TOOL_CALLS]get_weather[ARGS]{\"x\":1}", - &mistral3_markers(), - &shape, - ) - .expect("empty-separator shape must parse"); - - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_for_empty_body() { - let parsed = - parse("", &mistral3_markers(), &mistral3_shape()).expect("empty body must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_when_body_lacks_separator() { - let parsed = parse( - "plain text without separator", - &mistral3_markers(), - &mistral3_shape(), - ) - .expect("body without separator must parse"); - assert_eq!(parsed, Vec::::new()); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/consume_optional_prefix.rs b/llama-cpp-bindings/src/tool_call_format/consume_optional_prefix.rs deleted file mode 100644 index bc1ea817b..000000000 --- a/llama-cpp-bindings/src/tool_call_format/consume_optional_prefix.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[must_use] -pub fn consume_optional_prefix<'body>(input: &'body str, literal: &str) -> &'body str { - input.strip_prefix(literal).unwrap_or(input) -} diff --git a/llama-cpp-bindings/src/tool_call_format/json_object.rs b/llama-cpp-bindings/src/tool_call_format/json_object.rs deleted file mode 100644 index c82a322af..000000000 --- a/llama-cpp-bindings/src/tool_call_format/json_object.rs +++ /dev/null @@ -1,204 +0,0 @@ -use llama_cpp_bindings_types::JsonObjectShape; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; - -use crate::error::JsonObjectFailure; - -fn try_parse_one_object( - input: &str, - shape: &JsonObjectShape, -) -> Result, JsonObjectFailure> { - let trimmed_start = input.find('{'); - let Some(start) = trimmed_start else { - return Ok(None); - }; - - let mut stream = serde_json::Deserializer::from_str(&input[start..]) - .into_iter::>(); - let map = match stream.next() { - Some(Ok(map)) => map, - Some(Err(err)) => { - return Err(JsonObjectFailure::InvalidJson { - message: err.to_string(), - }); - } - None => return Ok(None), - }; - let consumed = stream.byte_offset(); - - let Some(name_value) = map.get(&shape.name_field) else { - return Ok(None); - }; - let serde_json::Value::String(name) = name_value else { - return Ok(None); - }; - - let arguments_value = map - .get(&shape.arguments_field) - .cloned() - .unwrap_or(serde_json::Value::Object(serde_json::Map::new())); - let arguments = ToolCallArguments::from_string(arguments_value.to_string()); - - let trailing_extras = map - .keys() - .any(|key| key != &shape.name_field && key != &shape.arguments_field); - if trailing_extras { - return Ok(None); - } - - Ok(Some(( - ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - name.clone(), - arguments, - ), - start + consumed, - ))) -} - -/// # Errors -/// -/// Returns [`JsonObjectFailure`] when the body contains a JSON object that -/// looks like a tool call (matches the open brace at start) but the JSON itself -/// is malformed. -pub fn parse( - body: &str, - shape: &JsonObjectShape, -) -> Result, JsonObjectFailure> { - if shape.name_field.is_empty() || shape.arguments_field.is_empty() { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body; - - while let Some((call, consumed)) = try_parse_one_object(remaining, shape)? { - parsed.push(call); - remaining = &remaining[consumed..]; - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::JsonObjectShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ToolCallArguments; - use serde_json::json; - - use super::parse; - use crate::error::JsonObjectFailure; - - fn qwen3_shape() -> JsonObjectShape { - JsonObjectShape { - name_field: "name".to_owned(), - arguments_field: "arguments".to_owned(), - } - } - - #[test] - fn parses_single_json_object_with_name_and_arguments() { - let parsed = parse( - r#"{"name": "get_weather", "arguments": {"location": "Paris"}}"#, - &qwen3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_json_object_after_leading_whitespace_and_newlines() { - let parsed = parse( - "\n {\"name\": \"f\", \"arguments\": {\"a\": 1}}\n", - &qwen3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "f"); - } - - #[test] - fn parses_two_consecutive_json_objects() { - let parsed = parse( - r#"{"name": "a", "arguments": {}}{"name": "b", "arguments": {"x": 2}}"#, - &qwen3_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 2); - assert_eq!(parsed[0].name, "a"); - assert_eq!(parsed[1].name, "b"); - } - - #[test] - fn parses_object_with_arguments_field_missing_yields_empty_arguments() { - let parsed = parse(r#"{"name": "ping"}"#, &qwen3_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "ping"); - assert_eq!(parsed[0].arguments, ToolCallArguments::ValidJson(json!({})),); - } - - #[test] - fn rejects_json_object_with_extra_unexpected_top_level_keys() { - let parsed = parse( - r#"{"name": "f", "arguments": {}, "extra": 1}"#, - &qwen3_shape(), - ) - .expect("must parse"); - - assert!(parsed.is_empty(), "extra top-level key must reject"); - } - - #[test] - fn rejects_json_object_with_non_string_name() { - let parsed = - parse(r#"{"name": 123, "arguments": {}}"#, &qwen3_shape()).expect("must parse"); - - assert!(parsed.is_empty(), "non-string name must reject"); - } - - #[test] - fn rejects_input_without_open_brace() { - let parsed = parse("plain content", &qwen3_shape()).expect("must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn rejects_array_instead_of_object() { - let parsed = parse("[1, 2, 3]", &qwen3_shape()).expect("must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_failure_for_malformed_json() { - let err = parse(r#"{"name": "f", "arguments": {"a": }"#, &qwen3_shape()).unwrap_err(); - let JsonObjectFailure::InvalidJson { message } = err; - - assert_ne!(message, ""); - } - - #[test] - fn returns_empty_when_object_is_not_a_tool_call_shape() { - let parsed = parse("{ \"foo\": 1 }", &qwen3_shape()).expect("must parse"); - - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_when_shape_has_empty_required_field() { - let mut shape = qwen3_shape(); - shape.name_field.clear(); - let parsed = parse(r#"{"name": "x", "arguments": {}}"#, &shape).expect("must parse"); - assert_eq!(parsed, Vec::::new()); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs b/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs deleted file mode 100644 index 55dcb3233..000000000 --- a/llama-cpp-bindings/src/tool_call_format/key_value_xml_tags.rs +++ /dev/null @@ -1,337 +0,0 @@ -use llama_cpp_bindings_types::KeyValueXmlTagsShape; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_bindings_types::ToolCallMarkers; -use nom::IResult; -use nom::Parser; -use nom::bytes::complete::take_until; - -use crate::error::KeyValueXmlTagsFailure; -use crate::tool_call_format::parse_step::ParseStep; -use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; - -const fn shape_is_complete(shape: &KeyValueXmlTagsShape) -> bool { - !shape.key_open.is_empty() - && !shape.key_close.is_empty() - && !shape.value_open.is_empty() - && !shape.value_close.is_empty() -} - -fn skip_to_next_open<'body>(input: &'body str, open: &str) -> Option<&'body str> { - let take_result: IResult<&'body str, &'body str> = take_until(open).parse(input); - let (after_open_inclusive, _) = take_result.ok()?; - - Some(&after_open_inclusive[open.len()..]) -} - -fn parse_one_parameter<'body>( - input: &'body str, - shape: &KeyValueXmlTagsShape, - function_name: &str, -) -> Result, KeyValueXmlTagsFailure> { - let take_result: IResult<&'body str, &'body str> = - take_until(shape.key_open.as_str()).parse(input); - let Ok((after_key_open_inclusive, _)) = take_result else { - return Ok(None); - }; - let after_key_open = &after_key_open_inclusive[shape.key_open.len()..]; - - let key_close_position = after_key_open - .find(shape.key_close.as_str()) - .ok_or_else(|| KeyValueXmlTagsFailure::UnclosedKeyTag { - function_name: function_name.to_owned(), - expected_close: shape.key_close.clone(), - })?; - let key = after_key_open[..key_close_position].trim().to_owned(); - if key.is_empty() { - return Err(KeyValueXmlTagsFailure::EmptyKey { - function_name: function_name.to_owned(), - }); - } - let after_key_close = &after_key_open[key_close_position + shape.key_close.len()..]; - - let value_open_take: IResult<&str, &str> = - take_until(shape.value_open.as_str()).parse(after_key_close); - let Ok((after_value_open_inclusive, _)) = value_open_take else { - return Err(KeyValueXmlTagsFailure::MissingValueTag { - function_name: function_name.to_owned(), - key, - expected_open: shape.value_open.clone(), - }); - }; - let after_value_open = &after_value_open_inclusive[shape.value_open.len()..]; - - let value_close_position = after_value_open - .find(shape.value_close.as_str()) - .ok_or_else(|| KeyValueXmlTagsFailure::UnclosedValueTag { - function_name: function_name.to_owned(), - key: key.clone(), - expected_close: shape.value_close.clone(), - })?; - let raw_value = &after_value_open[..value_close_position]; - let value = scalar_value_to_json(raw_value); - let after_value_close = &after_value_open[value_close_position + shape.value_close.len()..]; - - Ok(Some((key, value, after_value_close))) -} - -fn collect_parameters( - function_body: &str, - shape: &KeyValueXmlTagsShape, - function_name: &str, -) -> Result, KeyValueXmlTagsFailure> { - let mut parameters = serde_json::Map::new(); - let mut remaining = function_body; - - while let Some((key, value, rest)) = parse_one_parameter(remaining, shape, function_name)? { - parameters.insert(key, value); - remaining = rest; - } - - Ok(parameters) -} - -fn parse_one_call<'body>( - input: &'body str, - markers: &ToolCallMarkers, - shape: &KeyValueXmlTagsShape, -) -> Result, KeyValueXmlTagsFailure> { - let Some(after_open) = skip_to_next_open(input, &markers.open) else { - return Ok(ParseStep::Done); - }; - - let Some(close_position) = after_open.find(markers.close.as_str()) else { - return Err(KeyValueXmlTagsFailure::UnclosedFunctionBlock { - expected_close: markers.close.clone(), - }); - }; - let function_block = &after_open[..close_position]; - let after_function_close = &after_open[close_position + markers.close.len()..]; - - let (name_end, has_args) = function_block - .find(shape.key_open.as_str()) - .map_or((function_block.len(), false), |position| (position, true)); - let function_name = function_block[..name_end].trim().to_owned(); - if function_name.is_empty() { - return Err(KeyValueXmlTagsFailure::EmptyFunctionName); - } - - let args_section = if has_args { - &function_block[name_end..] - } else { - "" - }; - let arguments_object = collect_parameters(args_section, shape, &function_name)?; - let arguments_value = serde_json::Value::Object(arguments_object); - let arguments = ToolCallArguments::from_string(arguments_value.to_string()); - - Ok(ParseStep::Call { - call: ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - function_name, - arguments, - ), - remainder: after_function_close, - }) -} - -/// # Errors -/// -/// Returns [`KeyValueXmlTagsFailure`] when the body looks like a key-value-XML -/// tool-call block (matches the open marker) but contains a structural issue: -/// empty function/key name, missing key/value tag, or unclosed function block. -pub fn parse( - body: &str, - markers: &ToolCallMarkers, - shape: &KeyValueXmlTagsShape, -) -> Result, KeyValueXmlTagsFailure> { - if !shape_is_complete(shape) || markers.open.is_empty() || markers.close.is_empty() { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body; - - loop { - match parse_one_call(remaining, markers, shape)? { - ParseStep::Done => break, - ParseStep::Call { call, remainder } => { - parsed.push(call); - remaining = remainder; - } - } - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::KeyValueXmlTagsShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::ToolCallMarkers; - use serde_json::json; - - use super::parse; - use crate::error::KeyValueXmlTagsFailure; - - fn glm47_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "".to_owned(), - close: "".to_owned(), - args_shape: ToolCallArgsShape::KeyValueXmlTags(glm47_shape()), - } - } - - fn glm47_shape() -> KeyValueXmlTagsShape { - KeyValueXmlTagsShape { - key_open: "".to_owned(), - key_close: "".to_owned(), - value_open: "".to_owned(), - value_close: "".to_owned(), - } - } - - #[test] - fn parses_single_call_with_one_argument() { - let body = "get_weatherlocationParis"; - let parsed = parse(body, &glm47_markers(), &glm47_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_call_with_multiple_arguments() { - let body = "set_thermostatroomkitchencelsius21"; - let parsed = parse(body, &glm47_markers(), &glm47_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "set_thermostat"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"room": "kitchen", "celsius": 21})), - ); - } - - #[test] - fn parses_two_calls_in_one_body() { - let body = "ax1by2"; - let parsed = parse(body, &glm47_markers(), &glm47_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 2); - assert_eq!(parsed[0].name, "a"); - assert_eq!(parsed[1].name, "b"); - } - - #[test] - fn parses_call_with_no_arguments() { - let body = "ping"; - let parsed = parse(body, &glm47_markers(), &glm47_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "ping"); - assert_eq!(parsed[0].arguments, ToolCallArguments::ValidJson(json!({})),); - } - - #[test] - fn rejects_unclosed_function_block_with_typed_failure() { - let body = "get_weatherlocationParis"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::UnclosedFunctionBlock { - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_empty_function_name_with_typed_failure() { - let body = "kv"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!(result, Err(KeyValueXmlTagsFailure::EmptyFunctionName)); - } - - #[test] - fn rejects_unclosed_key_tag_with_typed_failure() { - let body = "flocation"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::UnclosedKeyTag { - function_name: "f".to_owned(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_missing_value_tag_with_typed_failure() { - let body = "flocationParis"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::MissingValueTag { - function_name: "f".to_owned(), - key: "location".to_owned(), - expected_open: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_empty_key_with_typed_failure() { - let body = "fParis"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::EmptyKey { - function_name: "f".to_owned(), - }), - ); - } - - #[test] - fn rejects_unclosed_value_tag_with_typed_failure() { - let body = "flocationParis"; - let result = parse(body, &glm47_markers(), &glm47_shape()); - - assert_eq!( - result, - Err(KeyValueXmlTagsFailure::UnclosedValueTag { - function_name: "f".to_owned(), - key: "location".to_owned(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn returns_empty_for_body_without_open_marker() { - let parsed = - parse("plain text", &glm47_markers(), &glm47_shape()).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_when_shape_is_incomplete() { - let mut shape = glm47_shape(); - shape.value_close.clear(); - let body = "fkv"; - let parsed = parse(body, &glm47_markers(), &shape).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs deleted file mode 100644 index 0e1648563..000000000 --- a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs +++ /dev/null @@ -1,487 +0,0 @@ -use llama_cpp_bindings_types::PairedQuoteShape; -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_bindings_types::ToolCallMarkers; -use llama_cpp_bindings_types::ToolCallValueQuote; - -use crate::error::PairedQuoteFailure; -use crate::tool_call_format::consume_optional_prefix::consume_optional_prefix; -use crate::tool_call_format::parse_step::ParseStep; -use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; -use crate::tool_call_format::separator_split::SeparatorSplit; - -fn bare_value_to_json(text: &str) -> serde_json::Value { - if text.is_empty() { - return serde_json::Value::Null; - } - - scalar_value_to_json(text) -} - -fn find_bare_value_end(input: &str, close_marker: &str) -> usize { - for (byte_index, character) in input.char_indices() { - if character == ',' { - return byte_index; - } - if !close_marker.is_empty() && input[byte_index..].starts_with(close_marker) { - return byte_index; - } - } - - input.len() -} - -fn parse_one_key<'body>( - input: &'body str, - tool_name: &str, -) -> Result<(String, &'body str), PairedQuoteFailure> { - let Some((key_raw, after_colon)) = input.split_once(':') else { - return Err(PairedQuoteFailure::UnclosedArgumentBlock { - tool_name: tool_name.to_owned(), - state: "key", - }); - }; - let key = key_raw.trim().to_owned(); - if key.is_empty() { - return Err(PairedQuoteFailure::EmptyKey { - tool_name: tool_name.to_owned(), - }); - } - - Ok((key, after_colon)) -} - -fn parse_one_value<'body>( - input: &'body str, - value_quote: &ToolCallValueQuote, - close_marker: &str, - tool_name: &str, - key: &str, -) -> Result<(serde_json::Value, &'body str), PairedQuoteFailure> { - let trimmed = input.trim_start(); - - if !value_quote.open.is_empty() - && !value_quote.close.is_empty() - && let Some(after_open) = trimmed.strip_prefix(value_quote.open.as_str()) - { - let Some(close_position) = after_open.find(value_quote.close.as_str()) else { - return Err(PairedQuoteFailure::UnclosedQuotedValue { - tool_name: tool_name.to_owned(), - key: key.to_owned(), - }); - }; - let value_text = after_open[..close_position].to_owned(); - let after_close = &after_open[close_position + value_quote.close.len()..]; - - return Ok((serde_json::Value::String(value_text), after_close)); - } - - let bare_end = find_bare_value_end(trimmed, close_marker); - let bare_text = trimmed[..bare_end].trim(); - let value = bare_value_to_json(bare_text); - - Ok((value, &trimmed[bare_end..])) -} - -fn parse_args_body<'body>( - input: &'body str, - value_quote: &ToolCallValueQuote, - close_marker: &str, - tool_name: &str, -) -> Result<(serde_json::Map, &'body str), PairedQuoteFailure> { - let mut map = serde_json::Map::new(); - let mut remaining = input.trim_start(); - - loop { - if remaining.is_empty() { - return Ok((map, remaining)); - } - if !close_marker.is_empty() - && let Some(after_close) = remaining.strip_prefix(close_marker) - { - return Ok((map, after_close)); - } - - let (key, after_key) = parse_one_key(remaining, tool_name)?; - let (value, after_value) = - parse_one_value(after_key, value_quote, close_marker, tool_name, &key)?; - map.insert(key.clone(), value); - - remaining = after_value.trim_start(); - if !close_marker.is_empty() - && let Some(after_close) = remaining.strip_prefix(close_marker) - { - return Ok((map, after_close)); - } - if let Some(after_comma) = remaining.strip_prefix(',') { - remaining = after_comma.trim_start(); - continue; - } - - let Some(character) = remaining.chars().next() else { - return Ok((map, remaining)); - }; - - return Err(PairedQuoteFailure::UnexpectedCharAfterValue { - tool_name: tool_name.to_owned(), - key, - character, - }); - } -} - -fn parse_one_call<'body>( - input: &'body str, - markers: &ToolCallMarkers, - shape: &PairedQuoteShape, -) -> Result, PairedQuoteFailure> { - if input.is_empty() { - return Ok(ParseStep::Done); - } - - let after_open = consume_optional_prefix(input, markers.open.as_str()); - - let Some(SeparatorSplit { - before: name_raw, - after: after_separator, - }) = SeparatorSplit::at_first(after_open, shape.name_args_separator.as_str()) - else { - return Ok(ParseStep::Done); - }; - - let name = name_raw.trim().to_owned(); - if name.is_empty() { - return Ok(ParseStep::Done); - } - - let (args_object, after_args) = parse_args_body( - after_separator, - &shape.value_quote, - markers.close.as_str(), - &name, - )?; - let arguments_value = serde_json::Value::Object(args_object); - - Ok(ParseStep::Call { - call: ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - name, - ToolCallArguments::ValidJson(arguments_value), - ), - remainder: after_args, - }) -} - -/// # Errors -/// -/// Returns [`PairedQuoteFailure`] when the body looks like a paired-quote -/// tool-call block (matches the open marker and separator) but contains a -/// structural issue: empty key, unclosed quoted value, unexpected character -/// after a value, or an unfinished argument block. -pub fn parse( - body: &str, - markers: &ToolCallMarkers, - shape: &PairedQuoteShape, -) -> Result, PairedQuoteFailure> { - if shape.name_args_separator.is_empty() { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body.trim_start(); - - loop { - match parse_one_call(remaining, markers, shape)? { - ParseStep::Done => break, - ParseStep::Call { call, remainder } => { - parsed.push(call); - remaining = remainder.trim_start(); - } - } - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::PairedQuoteShape; - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ToolCallArgsShape; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::ToolCallMarkers; - use llama_cpp_bindings_types::ToolCallValueQuote; - use serde_json::json; - - use super::parse; - use crate::error::PairedQuoteFailure; - - fn gemma4_markers() -> ToolCallMarkers { - ToolCallMarkers { - open: "<|tool_call>call:".to_owned(), - close: "}".to_owned(), - args_shape: ToolCallArgsShape::PairedQuote(gemma4_shape()), - } - } - - fn gemma4_shape() -> PairedQuoteShape { - PairedQuoteShape { - name_args_separator: "{".to_owned(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - } - } - - fn gemma4_call(arguments_body: &str) -> String { - let mut call = String::from("<|tool_call>call:f"); - call.push('{'); - call.push_str(arguments_body); - call.push('}'); - - call - } - - #[test] - fn parses_single_quoted_string_argument_with_full_markers() { - let parsed = parse( - "<|tool_call>call:get_weather{location:<|\"|>Paris<|\"|>}", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_classifier_stripped_body_without_open_or_close() { - let parsed = parse( - "get_weather{location:<|\"|>Paris<|\"|>", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_multiple_quoted_string_arguments() { - let parsed = parse( - "<|tool_call>call:f{a:<|\"|>1<|\"|>,b:<|\"|>2<|\"|>}", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": "1", "b": "2"})), - ); - } - - #[test] - fn parses_bare_numeric_value() { - let parsed = - parse(&gemma4_call("a:42"), &gemma4_markers(), &gemma4_shape()).expect("must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": 42})), - ); - } - - #[test] - fn parses_bare_boolean_value() { - let parsed = - parse(&gemma4_call("a:true"), &gemma4_markers(), &gemma4_shape()).expect("must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": true})), - ); - } - - #[test] - fn rejects_unclosed_quoted_value_with_typed_failure() { - let result = parse( - "<|tool_call>call:f{a:<|\"|>oops", - &gemma4_markers(), - &gemma4_shape(), - ); - - assert_eq!( - result.expect_err("unclosed quote must produce a typed failure"), - PairedQuoteFailure::UnclosedQuotedValue { - tool_name: "f".to_owned(), - key: "a".to_owned(), - }, - ); - } - - #[test] - fn rejects_unexpected_char_after_value_with_typed_failure() { - let result = parse( - "<|tool_call>call:f{a:<|\"|>v<|\"|>$bad}", - &gemma4_markers(), - &gemma4_shape(), - ); - - assert_eq!( - result.expect_err("garbage after value must produce a typed failure"), - PairedQuoteFailure::UnexpectedCharAfterValue { - tool_name: "f".to_owned(), - key: "a".to_owned(), - character: '$', - }, - ); - } - - #[test] - fn returns_empty_vec_for_empty_body() { - let parsed = parse("", &gemma4_markers(), &gemma4_shape()).expect("empty body must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_when_body_lacks_separator() { - let parsed = parse("no separator anywhere", &gemma4_markers(), &gemma4_shape()) - .expect("body without separator must parse"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn parses_args_body_terminated_by_end_of_input_after_quoted_value() { - let parsed = parse( - "<|tool_call>call:f{x:<|\"|>v<|\"|>", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("end-of-input after quoted value must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"x": "v"})), - ); - } - - #[test] - fn parses_args_body_terminated_by_end_of_input_after_bare_value() { - let parsed = parse( - "<|tool_call>call:f{n:42", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("end-of-input after bare value must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"n": 42})), - ); - } - - #[test] - fn rejects_empty_key_with_typed_failure() { - let result = parse(&gemma4_call(":42"), &gemma4_markers(), &gemma4_shape()); - - assert_eq!( - result.expect_err("empty key must produce a typed failure"), - PairedQuoteFailure::EmptyKey { - tool_name: "f".to_owned(), - }, - ); - } - - #[test] - fn rejects_args_body_without_key_colon_with_typed_failure() { - let result = parse( - "<|tool_call>call:f{noColonHere", - &gemma4_markers(), - &gemma4_shape(), - ); - - assert_eq!( - result.expect_err("args body without colon must produce a typed failure"), - PairedQuoteFailure::UnclosedArgumentBlock { - tool_name: "f".to_owned(), - state: "key", - }, - ); - } - - #[test] - fn parses_empty_bare_value_as_null() { - let parsed = parse(&gemma4_call("a:"), &gemma4_markers(), &gemma4_shape()) - .expect("empty bare value must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": null})), - ); - } - - #[test] - fn parses_call_with_empty_args_body_terminated_by_end_of_input() { - let parsed = parse("<|tool_call>call:f{", &gemma4_markers(), &gemma4_shape()) - .expect("empty args body must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "f"); - assert_eq!(parsed[0].arguments, ToolCallArguments::ValidJson(json!({})),); - } - - #[test] - fn parses_call_with_empty_args_body_closed_by_marker() { - let parsed = parse("<|tool_call>call:f{}", &gemma4_markers(), &gemma4_shape()) - .expect("empty args body closed by marker must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "f"); - assert_eq!(parsed[0].arguments, ToolCallArguments::ValidJson(json!({})),); - } - - #[test] - fn stops_parsing_when_tool_name_is_empty() { - let parsed = parse( - "<|tool_call>call:{a:<|\"|>v<|\"|>}", - &gemma4_markers(), - &gemma4_shape(), - ) - .expect("empty tool name must yield no calls"); - - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_vec_when_separator_is_empty() { - let shape = PairedQuoteShape { - name_args_separator: String::new(), - value_quote: ToolCallValueQuote { - open: "<|\"|>".to_owned(), - close: "<|\"|>".to_owned(), - }, - }; - let parsed = parse( - "<|tool_call>call:f{a:<|\"|>v<|\"|>}", - &gemma4_markers(), - &shape, - ) - .expect("empty separator must yield no calls"); - - assert_eq!(parsed, Vec::::new()); - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/parse_step.rs b/llama-cpp-bindings/src/tool_call_format/parse_step.rs deleted file mode 100644 index 70eb59711..000000000 --- a/llama-cpp-bindings/src/tool_call_format/parse_step.rs +++ /dev/null @@ -1,10 +0,0 @@ -use llama_cpp_bindings_types::ParsedToolCall; - -#[derive(Debug)] -pub enum ParseStep<'body> { - Done, - Call { - call: ParsedToolCall, - remainder: &'body str, - }, -} diff --git a/llama-cpp-bindings/src/tool_call_format/scalar_value_to_json.rs b/llama-cpp-bindings/src/tool_call_format/scalar_value_to_json.rs deleted file mode 100644 index 6efe2fc98..000000000 --- a/llama-cpp-bindings/src/tool_call_format/scalar_value_to_json.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[must_use] -pub fn scalar_value_to_json(raw: &str) -> serde_json::Value { - serde_json::from_str::(raw) - .unwrap_or_else(|_not_json| serde_json::Value::String(raw.to_owned())) -} diff --git a/llama-cpp-bindings/src/tool_call_format/separator_split.rs b/llama-cpp-bindings/src/tool_call_format/separator_split.rs deleted file mode 100644 index 3f9eca60b..000000000 --- a/llama-cpp-bindings/src/tool_call_format/separator_split.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[derive(Debug, Eq, PartialEq)] -pub struct SeparatorSplit<'body> { - pub before: &'body str, - pub after: &'body str, -} - -impl<'body> SeparatorSplit<'body> { - #[must_use] - pub fn at_first(input: &'body str, separator: &str) -> Option { - input - .split_once(separator) - .map(|(before, after)| Self { before, after }) - } -} diff --git a/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs b/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs deleted file mode 100644 index 1791071cf..000000000 --- a/llama-cpp-bindings/src/tool_call_format/tool_call_format_outcome.rs +++ /dev/null @@ -1,10 +0,0 @@ -use llama_cpp_bindings_types::ParsedToolCall; - -use crate::error::ToolCallFormatFailure; - -#[derive(Debug, Eq, PartialEq)] -pub enum ToolCallFormatOutcome { - Parsed(Vec), - NoMatch, - Failed(ToolCallFormatFailure), -} diff --git a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs b/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs deleted file mode 100644 index 7fe6eb96a..000000000 --- a/llama-cpp-bindings/src/tool_call_format/xml_function_tags.rs +++ /dev/null @@ -1,367 +0,0 @@ -use llama_cpp_bindings_types::ParsedToolCall; -use llama_cpp_bindings_types::ParsedToolCallId; -use llama_cpp_bindings_types::ToolCallArguments; -use llama_cpp_bindings_types::XmlTagsShape; -use nom::IResult; -use nom::Parser; -use nom::bytes::complete::take_until; - -use crate::error::XmlFunctionTagsFailure; -use crate::tool_call_format::scalar_value_to_json::scalar_value_to_json; - -const fn shape_is_complete(shape: &XmlTagsShape) -> bool { - !shape.function_open_prefix.is_empty() - && !shape.function_close.is_empty() - && !shape.parameter_open_prefix.is_empty() - && !shape.parameter_close.is_empty() -} - -fn trim_surrounding_newlines(input: &str) -> &str { - input.trim_start_matches('\n').trim_end_matches('\n') -} - -fn locate_tag_name_end(after_prefix: &str) -> Option { - let close_position = after_prefix.find('>'); - let next_open_position = after_prefix.find('<'); - - match (close_position, next_open_position) { - (Some(close), Some(open)) if open < close => None, - (Some(close), _) => Some(close), - (None, _) => None, - } -} - -fn skip_to_next_function_open<'body>( - input: &'body str, - function_open_prefix: &str, -) -> Option<&'body str> { - let take_result: IResult<&'body str, &'body str> = - take_until(function_open_prefix).parse(input); - let (after_prefix_inclusive, _) = take_result.ok()?; - - Some(&after_prefix_inclusive[function_open_prefix.len()..]) -} - -fn parse_one_parameter<'body>( - input: &'body str, - shape: &XmlTagsShape, - function_name: &str, -) -> Result, XmlFunctionTagsFailure> { - let take_result: IResult<&'body str, &'body str> = - take_until(shape.parameter_open_prefix.as_str()).parse(input); - let Ok((after_prefix_inclusive, _)) = take_result else { - return Ok(None); - }; - let after_prefix = &after_prefix_inclusive[shape.parameter_open_prefix.len()..]; - - let Some(name_end) = locate_tag_name_end(after_prefix) else { - return Err(XmlFunctionTagsFailure::UnclosedParameterBlock { - function_name: function_name.to_owned(), - parameter_name: String::new(), - expected_close: shape.parameter_close.clone(), - }); - }; - let parameter_name = after_prefix[..name_end].trim().to_owned(); - if parameter_name.is_empty() { - return Err(XmlFunctionTagsFailure::EmptyParameterName { - function_name: function_name.to_owned(), - }); - } - let value_start = &after_prefix[name_end + 1..]; - - let Some(value_end_position) = value_start.find(shape.parameter_close.as_str()) else { - return Err(XmlFunctionTagsFailure::UnclosedParameterBlock { - function_name: function_name.to_owned(), - parameter_name, - expected_close: shape.parameter_close.clone(), - }); - }; - let raw_value = trim_surrounding_newlines(&value_start[..value_end_position]); - let after_close = &value_start[value_end_position + shape.parameter_close.len()..]; - let parameter_value = scalar_value_to_json(raw_value); - - Ok(Some((parameter_name, parameter_value, after_close))) -} - -fn collect_parameters( - function_body: &str, - shape: &XmlTagsShape, - function_name: &str, -) -> Result, XmlFunctionTagsFailure> { - let mut parameters = serde_json::Map::new(); - let mut remaining = function_body; - - while let Some((parameter_name, parameter_value, rest)) = - parse_one_parameter(remaining, shape, function_name)? - { - parameters.insert(parameter_name, parameter_value); - remaining = rest; - } - - Ok(parameters) -} - -fn parse_one_function<'body>( - input: &'body str, - shape: &XmlTagsShape, -) -> Result, XmlFunctionTagsFailure> { - let Some(after_function_prefix) = - skip_to_next_function_open(input, &shape.function_open_prefix) - else { - return Ok(None); - }; - - let Some(name_end) = locate_tag_name_end(after_function_prefix) else { - return Err(XmlFunctionTagsFailure::UnclosedFunctionBlock { - function_name: String::new(), - expected_close: shape.function_close.clone(), - }); - }; - let function_name = after_function_prefix[..name_end].trim().to_owned(); - if function_name.is_empty() { - return Err(XmlFunctionTagsFailure::EmptyFunctionName); - } - let function_body_start = &after_function_prefix[name_end + 1..]; - - let Some(function_body_end) = function_body_start.find(shape.function_close.as_str()) else { - return Err(XmlFunctionTagsFailure::UnclosedFunctionBlock { - function_name, - expected_close: shape.function_close.clone(), - }); - }; - let function_body = &function_body_start[..function_body_end]; - let after_function_close = - &function_body_start[function_body_end + shape.function_close.len()..]; - - let arguments_object = collect_parameters(function_body, shape, &function_name)?; - let arguments_value = serde_json::Value::Object(arguments_object); - let arguments = ToolCallArguments::from_string(arguments_value.to_string()); - - Ok(Some(( - ParsedToolCall::new( - ParsedToolCallId::NotProvidedByModel, - function_name, - arguments, - ), - after_function_close, - ))) -} - -/// # Errors -/// -/// Returns [`XmlFunctionTagsFailure`] when the body looks like an XML -/// function-tag tool-call block (matches the function open prefix) but -/// contains a structural issue: empty function/parameter name or an -/// unclosed function/parameter block. -pub fn parse( - body: &str, - shape: &XmlTagsShape, -) -> Result, XmlFunctionTagsFailure> { - if !shape_is_complete(shape) { - return Ok(Vec::new()); - } - - let mut parsed = Vec::new(); - let mut remaining = body; - - while let Some((call, rest)) = parse_one_function(remaining, shape)? { - parsed.push(call); - remaining = rest; - } - - Ok(parsed) -} - -#[cfg(test)] -mod tests { - use llama_cpp_bindings_types::ParsedToolCall; - use llama_cpp_bindings_types::ToolCallArguments; - use llama_cpp_bindings_types::XmlTagsShape; - use serde_json::json; - - use super::parse; - use crate::error::XmlFunctionTagsFailure; - - fn xml_shape() -> XmlTagsShape { - XmlTagsShape { - function_open_prefix: "".to_owned(), - parameter_open_prefix: "".to_owned(), - } - } - - #[test] - fn parses_single_function_with_one_parameter() { - let body = - "\n\n\nParis\n\n\n"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "get_weather"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"location": "Paris"})), - ); - } - - #[test] - fn parses_function_with_multiple_parameters() { - let body = "1two"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"a": 1, "b": "two"})), - ); - } - - #[test] - fn parses_two_function_blocks_in_one_body() { - let body = "12"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 2); - assert_eq!(parsed[0].name, "a"); - assert_eq!(parsed[1].name, "b"); - } - - #[test] - fn preserves_multi_line_parameter_value() { - let body = "\n\nline one\nline two\n\n"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({"msg": "line one\nline two"})), - ); - } - - #[test] - fn rejects_function_tag_missing_closing_angle_with_typed_failure() { - let body = "Paris"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::UnclosedFunctionBlock { - function_name: String::new(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_function_block_missing_close_tag_with_typed_failure() { - let body = "Paris"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::UnclosedFunctionBlock { - function_name: "get_weather".to_owned(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_parameter_tag_missing_closing_angle_with_typed_failure() { - let body = ""; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::UnclosedParameterBlock { - function_name: "f".to_owned(), - parameter_name: String::new(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_parameter_block_missing_close_tag_with_typed_failure() { - let body = "Paris"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::UnclosedParameterBlock { - function_name: "get_weather".to_owned(), - parameter_name: "location".to_owned(), - expected_close: "".to_owned(), - }), - ); - } - - #[test] - fn rejects_empty_function_name_with_typed_failure() { - let body = "1"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::EmptyFunctionName), - ); - } - - #[test] - fn rejects_empty_parameter_name_with_typed_failure() { - let body = "1"; - - assert_eq!( - parse(body, &xml_shape()), - Err(XmlFunctionTagsFailure::EmptyParameterName { - function_name: "f".to_owned(), - }), - ); - } - - #[test] - fn returns_empty_when_body_has_no_function_tag() { - let parsed = - parse("plain text without function tags", &xml_shape()).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_for_empty_body() { - let parsed = parse("", &xml_shape()).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn returns_empty_when_shape_has_empty_required_field() { - let mut shape = xml_shape(); - shape.function_close.clear(); - let body = "1"; - let parsed = parse(body, &shape).expect("must parse empty"); - assert_eq!(parsed, Vec::::new()); - } - - #[test] - fn parses_negotiate_with_cat_reproducer_payload() { - let body = "\n\ -\n\ -\n\ -tuna\n\ -\n\ -\n\ -8\n\ -\n\ -\n\ -get off the keyboard\n\ -\n\ -\n\ -"; - let parsed = parse(body, &xml_shape()).expect("must parse"); - - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].name, "negotiate_with_cat"); - assert_eq!( - parsed[0].arguments, - ToolCallArguments::ValidJson(json!({ - "bribe": "tuna", - "desperation_level": 8, - "topic": "get off the keyboard", - })), - ); - } -} diff --git a/llama-cpp-bindings/src/tool_call_marker_pair.rs b/llama-cpp-bindings/src/tool_call_marker_pair.rs deleted file mode 100644 index 3ee5fd42b..000000000 --- a/llama-cpp-bindings/src/tool_call_marker_pair.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ToolCallMarkerPair { - pub open: String, - pub close: String, -} diff --git a/llama-cpp-bindings/tests/sampler_chain_without_backend.rs b/llama-cpp-bindings/tests/sampler_chain_without_backend.rs index f8ff4ee0c..4dce40654 100644 --- a/llama-cpp-bindings/tests/sampler_chain_without_backend.rs +++ b/llama-cpp-bindings/tests/sampler_chain_without_backend.rs @@ -1,4 +1,5 @@ use anyhow::Result; + use llama_cpp_bindings::sampling::LlamaSampler; use llama_cpp_bindings::token::LlamaToken; use llama_cpp_bindings::token::data::LlamaTokenData; @@ -6,7 +7,7 @@ use llama_cpp_bindings::token::data_array::LlamaTokenDataArray; #[test] fn sampler_chain_applies_without_initialized_backend() -> Result<()> { - let sampler_chain = LlamaSampler::chain_simple([LlamaSampler::greedy()?])?; + let sampler_chain = LlamaSampler::chain([LlamaSampler::greedy()?])?; let mut candidates = LlamaTokenDataArray::new( vec![ LlamaTokenData::new(LlamaToken::new(0), 1.0, 0.0), @@ -16,7 +17,7 @@ fn sampler_chain_applies_without_initialized_backend() -> Result<()> { false, ); - candidates.apply_sampler(&sampler_chain)?; + sampler_chain.apply(&mut candidates)?; assert_eq!(candidates.selected_token(), Some(LlamaToken::new(1))); diff --git a/llama-cpp-error-recorder/src/error_scope.rs b/llama-cpp-error-recorder/src/error_scope.rs index ef018c2e1..7c21d7e40 100644 --- a/llama-cpp-error-recorder/src/error_scope.rs +++ b/llama-cpp-error-recorder/src/error_scope.rs @@ -37,17 +37,6 @@ mod tests { use crate::record::record; use crate::recorded_error::RecordedError; - #[test] - fn records_and_takes_within_a_scope() { - let scope = ErrorScope::enter(); - record(RecordedError::new("boom".to_string())); - - assert_eq!( - scope.take().map(RecordedError::into_message), - Some("boom".to_string()) - ); - } - #[test] fn keeps_the_first_recorded_error() { let scope = ErrorScope::enter(); @@ -72,7 +61,10 @@ mod tests { let scope = ErrorScope::enter(); record(RecordedError::new("once".to_string())); - assert!(scope.take().is_some()); + assert_eq!( + scope.take().map(RecordedError::into_message), + Some("once".to_string()) + ); assert!(scope.take().is_none()); } diff --git a/llama-cpp-error-recorder/src/frame_stack.rs b/llama-cpp-error-recorder/src/frame_stack.rs index e011c66da..f20262b12 100644 --- a/llama-cpp-error-recorder/src/frame_stack.rs +++ b/llama-cpp-error-recorder/src/frame_stack.rs @@ -1,5 +1,7 @@ use std::cell::RefCell; +use log::error; + use crate::recorded_error::RecordedError; thread_local! { @@ -27,15 +29,13 @@ impl FrameStack { FRAMES.with(|cell| { let mut frames = cell.borrow_mut(); let Some(top) = frames.last_mut() else { - log::error!("an FFI callback failed outside any error scope: {error}"); + error!("an FFI callback failed outside any error scope: {error}"); return; }; if top.is_some() { - log::error!( - "an FFI callback failed again after the root cause was recorded: {error}" - ); + error!("an FFI callback failed again after the root cause was recorded: {error}"); return; } diff --git a/llama-cpp-error-recorder/src/lib.rs b/llama-cpp-error-recorder/src/lib.rs index 42dcf41a8..b7a8e7e7e 100644 --- a/llama-cpp-error-recorder/src/lib.rs +++ b/llama-cpp-error-recorder/src/lib.rs @@ -8,7 +8,3 @@ mod frame_stack; pub mod error_scope; pub mod record; pub mod recorded_error; - -pub use error_scope::ErrorScope; -pub use record::record; -pub use recorded_error::RecordedError; diff --git a/llama-cpp-fault-injection/Cargo.toml b/llama-cpp-fault-injection/Cargo.toml new file mode 100644 index 000000000..0e3afbbb2 --- /dev/null +++ b/llama-cpp-fault-injection/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "llama-cpp-fault-injection" +description = "Arms the C++ wrapper fault points so tests can drive the paths llama.cpp only takes when it fails" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +llama-cpp-bindings-sys = { workspace = true, features = ["fault-injection"] } + +[lints] +workspace = true diff --git a/llama-cpp-fault-injection/src/armed_fault.rs b/llama-cpp-fault-injection/src/armed_fault.rs new file mode 100644 index 000000000..dce628b6a --- /dev/null +++ b/llama-cpp-fault-injection/src/armed_fault.rs @@ -0,0 +1,36 @@ +use std::marker::PhantomData; + +/// Makes one C++ wrapper fault point fail on the current thread until the guard is dropped. +/// +/// The fault point is armed per thread, so the guard cannot leave the thread that armed it. +#[derive(Debug)] +pub struct ArmedFault { + armed_on_this_thread: PhantomData<*const ()>, +} + +impl ArmedFault { + /// Fails every time the wrappers reach `point`. + #[must_use] + pub fn at(point: llama_cpp_bindings_sys::llama_rs_fault_point) -> Self { + Self::after_passing(point, 0) + } + + /// Lets the wrappers pass `point` `passing_hits` times, then fails every later time. + #[must_use] + pub fn after_passing( + point: llama_cpp_bindings_sys::llama_rs_fault_point, + passing_hits: usize, + ) -> Self { + unsafe { llama_cpp_bindings_sys::llama_rs_fault_injection_arm(point, passing_hits) }; + + Self { + armed_on_this_thread: PhantomData, + } + } +} + +impl Drop for ArmedFault { + fn drop(&mut self) { + unsafe { llama_cpp_bindings_sys::llama_rs_fault_injection_disarm() }; + } +} diff --git a/llama-cpp-fault-injection/src/injected_fault_message.rs b/llama-cpp-fault-injection/src/injected_fault_message.rs new file mode 100644 index 000000000..aefddd74e --- /dev/null +++ b/llama-cpp-fault-injection/src/injected_fault_message.rs @@ -0,0 +1,14 @@ +use std::ffi::CStr; + +/// The message of the C++ exception an armed fault point throws. +/// +/// # Panics +/// +/// Panics when the wrappers' fault message is not UTF-8. +#[must_use] +pub fn injected_fault_message() -> String { + unsafe { CStr::from_ptr(llama_cpp_bindings_sys::llama_rs_fault_injection_message()) } + .to_str() + .expect("the wrappers' fault message must be UTF-8") + .to_owned() +} diff --git a/llama-cpp-fault-injection/src/lib.rs b/llama-cpp-fault-injection/src/lib.rs new file mode 100644 index 000000000..18526e8b3 --- /dev/null +++ b/llama-cpp-fault-injection/src/lib.rs @@ -0,0 +1,2 @@ +pub mod armed_fault; +pub mod injected_fault_message; diff --git a/llama-cpp-ffi-status/src/cpp_string_error.rs b/llama-cpp-ffi-status/src/cpp_string_error.rs new file mode 100644 index 000000000..ed1b355fd --- /dev/null +++ b/llama-cpp-ffi-status/src/cpp_string_error.rs @@ -0,0 +1,16 @@ +use std::str::Utf8Error; + +use crate::ffi_contract_error::FfiContractError; + +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +pub enum CppStringError { + #[error(transparent)] + Missing(#[from] FfiContractError), + #[error("{operation} reported text that is not valid UTF-8: {source}")] + NotUtf8 { + operation: &'static str, + bytes: Vec, + #[source] + source: Utf8Error, + }, +} diff --git a/llama-cpp-ffi-status/src/lib.rs b/llama-cpp-ffi-status/src/lib.rs index c1ab08546..473752b16 100644 --- a/llama-cpp-ffi-status/src/lib.rs +++ b/llama-cpp-ffi-status/src/lib.rs @@ -8,10 +8,7 @@ ) )] +pub mod cpp_string_error; pub mod ffi_contract_error; pub mod ffi_status_error; pub mod read_and_free_cpp_string; - -pub use ffi_contract_error::FfiContractError; -pub use ffi_status_error::FfiStatusError; -pub use read_and_free_cpp_string::read_and_free_cpp_string; diff --git a/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs b/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs index ab25f7d20..e5bd2aa08 100644 --- a/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs +++ b/llama-cpp-ffi-status/src/read_and_free_cpp_string.rs @@ -1,16 +1,26 @@ use std::ffi::CStr; use std::ffi::c_char; +use crate::cpp_string_error::CppStringError; use crate::ffi_contract_error::FfiContractError; +fn utf8_text_of(bytes: Vec, operation: &'static str) -> Result { + String::from_utf8(bytes).map_err(|not_utf8| CppStringError::NotUtf8 { + operation, + source: not_utf8.utf8_error(), + bytes: not_utf8.into_bytes(), + }) +} + /// Takes ownership of a C string a wrapper stored in one of its out-parameters and /// hands the allocation back to the wrapper. /// /// # Errors /// -/// Returns [`FfiContractError`] when `string_ptr` is null. A wrapper always fills the -/// slot it says it filled, so a null pointer means the wrapper broke its own contract -/// rather than that the string is unknown. +/// Returns [`CppStringError::Missing`] when `string_ptr` is null. A wrapper always fills the +/// slot it says it filled, so a null pointer means the wrapper broke its own contract rather +/// than that the string is unknown. Returns [`CppStringError::NotUtf8`], keeping the bytes, +/// when the string is not valid UTF-8. /// /// # Safety /// @@ -20,30 +30,30 @@ pub unsafe fn read_and_free_cpp_string( string_ptr: *mut c_char, operation: &'static str, detail_when_missing: &'static str, -) -> Result { +) -> Result { if string_ptr.is_null() { return Err(FfiContractError { operation, detail: detail_when_missing, - }); + } + .into()); } - let value = unsafe { CStr::from_ptr(string_ptr) } - .to_string_lossy() - .into_owned(); + let bytes = unsafe { CStr::from_ptr(string_ptr) }.to_bytes().to_vec(); unsafe { llama_cpp_bindings_sys::llama_rs_string_free(string_ptr) }; - Ok(value) + utf8_text_of(bytes, operation) } #[cfg(test)] mod tests { use std::ptr; - use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; use super::read_and_free_cpp_string; + use crate::cpp_string_error::CppStringError; use crate::ffi_contract_error::FfiContractError; #[test] @@ -57,8 +67,12 @@ mod tests { }; assert_eq!( - message.map(|text| text.contains("parse error")), - Ok(true), + message, + Ok( + "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error \ + while parsing value - invalid literal; last read: 'no'" + .to_owned() + ), "the llama.cpp json parser reports its failure through the error slot" ); } @@ -75,9 +89,30 @@ mod tests { assert_eq!( result, - Err(FfiContractError { + Err(CppStringError::Missing(FfiContractError { operation: "llama_rs_json_schema_to_grammar", detail: "reported a thrown C++ exception without an error message", + })) + ); + } + + #[test] + fn a_string_that_is_not_utf8_keeps_its_bytes() { + let not_utf8 = c"llama\xff"; + let result = unsafe { + read_and_free_cpp_string( + llama_cpp_bindings_sys::llama_rs_string_dup(not_utf8.as_ptr()), + "llama_rs_json_schema_to_grammar", + "reported a thrown C++ exception without an error message", + ) + }; + + assert_eq!( + result, + Err(CppStringError::NotUtf8 { + operation: "llama_rs_json_schema_to_grammar", + bytes: not_utf8.to_bytes().to_vec(), + source: not_utf8.to_str().expect_err("0xff is not UTF-8"), }) ); } diff --git a/llama-cpp-gbnf/src/gbnf_validation_error.rs b/llama-cpp-gbnf/src/gbnf_validation_error.rs index fe8891cd9..5f95531bb 100644 --- a/llama-cpp-gbnf/src/gbnf_validation_error.rs +++ b/llama-cpp-gbnf/src/gbnf_validation_error.rs @@ -1,11 +1,17 @@ use std::ffi::NulError; +use llama_cpp_ffi_status::cpp_string_error::CppStringError; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; + #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum GbnfValidationError { #[error(transparent)] - FfiStatus(#[from] llama_cpp_ffi_status::FfiStatusError), + CppString(#[from] CppStringError), + #[error(transparent)] + FfiStatus(#[from] FfiStatusError), #[error(transparent)] - FfiContract(#[from] llama_cpp_ffi_status::FfiContractError), + FfiContract(#[from] FfiContractError), #[error("grammar string contains an interior NUL byte")] GrammarContainsNul(#[source] NulError), #[error("grammar root name contains an interior NUL byte")] @@ -16,8 +22,10 @@ pub enum GbnfValidationError { EmptyRuleSet, #[error("grammar does not define the root symbol {root:?}")] RootSymbolMissing { root: String }, - #[error("grammar is left-recursive and cannot be compiled by llama.cpp")] - LeftRecursion, + #[error( + "grammar is left-recursive or chains rules deeper than llama.cpp's recursion limit, so llama.cpp cannot compile it" + )] + UnsupportedGrammar, #[error("not enough memory")] NotEnoughMemory, #[error("the llama.cpp library ran out of memory")] diff --git a/llama-cpp-gbnf/src/validate_gbnf.rs b/llama-cpp-gbnf/src/validate_gbnf.rs index 60f71e848..c15c02e49 100644 --- a/llama-cpp-gbnf/src/validate_gbnf.rs +++ b/llama-cpp-gbnf/src/validate_gbnf.rs @@ -1,19 +1,18 @@ use std::ffi::{CString, c_char}; use std::ptr; -use llama_cpp_ffi_status::FfiContractError; -use llama_cpp_ffi_status::FfiStatusError; -use llama_cpp_ffi_status::read_and_free_cpp_string; - use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_EMPTY_RULE_SET; -use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_OK; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_ROOT_SYMBOL_MISSING; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_SYNTAX_ERROR; use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; +use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR; use llama_cpp_bindings_sys::llama_rs_gbnf_validation_status; use llama_cpp_bindings_sys::llama_rs_validate_gbnf; use llama_cpp_bindings_sys::llama_vocab; +use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; +use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; +use llama_cpp_ffi_status::read_and_free_cpp_string::read_and_free_cpp_string; use crate::gbnf_validation_error::GbnfValidationError; @@ -31,7 +30,9 @@ fn validation_status_to_result( root: root.to_owned(), }) } - LLAMA_RS_GBNF_VALIDATION_LEFT_RECURSION => Err(GbnfValidationError::LeftRecursion), + LLAMA_RS_GBNF_VALIDATION_UNSUPPORTED_GRAMMAR => { + Err(GbnfValidationError::UnsupportedGrammar) + } llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_NULL_GRAMMAR_ARG => { Err(FfiContractError { operation: "llama_rs_validate_gbnf", @@ -103,15 +104,18 @@ pub fn validate_gbnf( #[cfg(test)] mod tests { + use llama_cpp_ffi_status::cpp_string_error::CppStringError; use std::ffi::CString; + use std::fmt::Write; + use std::ptr; - use llama_cpp_ffi_status::FfiContractError; - use llama_cpp_ffi_status::FfiStatusError; + use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; + use llama_cpp_ffi_status::ffi_contract_error::FfiContractError; + use llama_cpp_ffi_status::ffi_status_error::FfiStatusError; use super::validate_gbnf; use super::validation_status_to_result; use crate::gbnf_validation_error::GbnfValidationError; - use llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION; #[test] fn valid_grammar_is_accepted() { @@ -151,7 +155,21 @@ mod tests { fn left_recursive_grammar_is_rejected() { assert_eq!( validate_gbnf(None, r#"root ::= root "x""#, "root"), - Err(GbnfValidationError::LeftRecursion) + Err(GbnfValidationError::UnsupportedGrammar) + ); + } + + #[test] + fn grammar_chaining_rules_past_the_recursion_limit_is_rejected() { + let mut grammar = String::from("root ::= rule0\n"); + for rule_index in 0..300 { + writeln!(grammar, "rule{rule_index} ::= rule{}", rule_index + 1).unwrap(); + } + grammar.push_str("rule300 ::= \"x\"\n"); + + assert_eq!( + validate_gbnf(None, &grammar, "root"), + Err(GbnfValidationError::UnsupportedGrammar) ); } @@ -180,17 +198,19 @@ mod tests { } #[test] - fn exception_status_without_message_maps_to_unknown_reported_error() { + fn exception_status_without_message_is_a_contract_error() { assert_eq!( validation_status_to_result( LLAMA_RS_GBNF_VALIDATION_THREW_CXX_EXCEPTION, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), - Err(GbnfValidationError::FfiContract(FfiContractError { - operation: "llama_rs_validate_gbnf", - detail: "reported a thrown C++ exception without an error message", - })) + Err(GbnfValidationError::CppString(CppStringError::Missing( + FfiContractError { + operation: "llama_rs_validate_gbnf", + detail: "reported a thrown C++ exception without an error message", + } + ))) ); } @@ -219,7 +239,7 @@ mod tests { validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_NULL_GRAMMAR_ARG, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::FfiContract(FfiContractError { operation: "llama_rs_validate_gbnf", @@ -234,7 +254,7 @@ mod tests { validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_NULL_ROOT_ARG, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::FfiContract(FfiContractError { operation: "llama_rs_validate_gbnf", @@ -249,7 +269,7 @@ mod tests { validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_NULL_OUT_ERROR_ARG, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::FfiContract(FfiContractError { operation: "llama_rs_validate_gbnf", @@ -259,12 +279,12 @@ mod tests { } #[test] - fn llama_cpp_out_of_memory_status_is_distinct_from_error_string_allocation() { + fn llama_cpp_out_of_memory_status_maps_to_llama_cpp_out_of_memory() { assert_eq!( validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_LLAMA_CPP_OUT_OF_MEMORY, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::LlamaCppOutOfMemory) ); @@ -276,7 +296,7 @@ mod tests { validation_status_to_result( llama_cpp_bindings_sys::LLAMA_RS_GBNF_VALIDATION_ERROR_STRING_ALLOCATION_FAILED, "root", - std::ptr::null_mut(), + ptr::null_mut(), ), Err(GbnfValidationError::NotEnoughMemory) ); @@ -285,7 +305,7 @@ mod tests { #[test] fn unknown_status_is_preserved() { assert_eq!( - validation_status_to_result(255, "root", std::ptr::null_mut(),), + validation_status_to_result(255, "root", ptr::null_mut(),), Err(GbnfValidationError::FfiStatus(FfiStatusError { operation: "llama_rs_validate_gbnf", code: 255, diff --git a/llama-cpp-log-decoder/src/incoming_log_level.rs b/llama-cpp-log-decoder/src/incoming_log_level.rs index 3cce5605c..9dc8cbce0 100644 --- a/llama-cpp-log-decoder/src/incoming_log_level.rs +++ b/llama-cpp-log-decoder/src/incoming_log_level.rs @@ -14,7 +14,7 @@ mod tests { use super::IncomingLogLevel; #[test] - fn unknown_variant_equality() { + fn unknown_levels_are_equal_only_with_the_same_raw_value() { assert_eq!(IncomingLogLevel::Unknown(42), IncomingLogLevel::Unknown(42)); assert_ne!(IncomingLogLevel::Unknown(42), IncomingLogLevel::Unknown(43)); } diff --git a/llama-cpp-log-decoder/src/log_decoder.rs b/llama-cpp-log-decoder/src/log_decoder.rs index fba0ba0a7..a4116af76 100644 --- a/llama-cpp-log-decoder/src/log_decoder.rs +++ b/llama-cpp-log-decoder/src/log_decoder.rs @@ -163,7 +163,7 @@ mod tests { use crate::log_line::LogLine; #[test] - fn feed_complete_info_line() { + fn a_complete_line_is_decoded_without_its_newline() { let mut decoder = LogDecoder::new(); let result = decoder.feed(IncomingLogLevel::Info, "hello\n"); @@ -180,7 +180,7 @@ mod tests { } #[test] - fn feed_partial_without_newline() { + fn a_line_without_a_newline_is_held_back() { let mut decoder = LogDecoder::new(); let result = decoder.feed(IncomingLogLevel::Info, "hello"); @@ -194,25 +194,7 @@ mod tests { } #[test] - fn feed_cont_completion() { - let mut decoder = LogDecoder::new(); - decoder.feed(IncomingLogLevel::Info, "hello "); - let result = decoder.feed(IncomingLogLevel::Cont, "world\n"); - - assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::Info, - text: "hello world".to_owned(), - }), - anomaly: None, - } - ); - } - - #[test] - fn feed_multi_part_cont() { + fn continuations_join_until_the_newline() { let mut decoder = LogDecoder::new(); decoder.feed(IncomingLogLevel::Info, "part1 "); decoder.feed(IncomingLogLevel::Cont, "part2 "); @@ -231,7 +213,7 @@ mod tests { } #[test] - fn feed_non_cont_while_buffering() { + fn a_new_complete_line_flushes_the_held_line_as_stale() { let mut decoder = LogDecoder::new(); decoder.feed(IncomingLogLevel::Info, "stale"); let result = decoder.feed(IncomingLogLevel::Warn, "fresh\n"); @@ -255,7 +237,7 @@ mod tests { } #[test] - fn feed_buffer_replacement() { + fn a_new_partial_line_flushes_the_held_line_and_is_held_in_its_place() { let mut decoder = LogDecoder::new(); decoder.feed(IncomingLogLevel::Info, "first"); let result = decoder.feed(IncomingLogLevel::Warn, "second"); @@ -285,7 +267,7 @@ mod tests { } #[test] - fn feed_orphan_cont() { + fn a_continuation_without_any_earlier_line_is_an_orphan_anomaly() { let mut decoder = LogDecoder::new(); let result = decoder.feed(IncomingLogLevel::Cont, "ghost\n"); @@ -302,7 +284,7 @@ mod tests { } #[test] - fn feed_orphan_cont_previous_level() { + fn an_orphan_continuation_takes_the_previous_line_level() { let mut decoder = LogDecoder::new(); decoder.feed(IncomingLogLevel::Warn, "complete\n"); let result = decoder.feed(IncomingLogLevel::Cont, "ghost\n"); @@ -320,70 +302,31 @@ mod tests { } #[test] - fn feed_none_level() { - let mut decoder = LogDecoder::new(); - let result = decoder.feed(IncomingLogLevel::None, "no-level\n"); - - assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::None, - text: "no-level".to_owned(), - }), - anomaly: None, - } - ); - } - - #[test] - fn feed_unknown_level() { - let mut decoder = LogDecoder::new(); - let result = decoder.feed(IncomingLogLevel::Unknown(9999), "weird\n"); + fn a_complete_line_keeps_the_level_it_arrived_with() { + let incoming_levels = [ + IncomingLogLevel::None, + IncomingLogLevel::Unknown(9999), + IncomingLogLevel::Debug, + IncomingLogLevel::Error, + ]; assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::Unknown(9999), - text: "weird".to_owned(), - }), - anomaly: None, - } - ); - } - - #[test] - fn feed_debug_level() { - let mut decoder = LogDecoder::new(); - let result = decoder.feed(IncomingLogLevel::Debug, "trace\n"); - - assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::Debug, - text: "trace".to_owned(), - }), - anomaly: None, - } - ); - } - - #[test] - fn feed_error_level() { - let mut decoder = LogDecoder::new(); - let result = decoder.feed(IncomingLogLevel::Error, "boom\n"); - - assert_eq!( - result, - DecodeResult { - output: DecodeOutput::Line(LogLine { - level: LogLevel::Error, - text: "boom".to_owned(), - }), - anomaly: None, - } + incoming_levels.map(|incoming_level| LogDecoder::new().feed(incoming_level, "line\n")), + [ + LogLevel::None, + LogLevel::Unknown(9999), + LogLevel::Debug, + LogLevel::Error + ] + .map(|level| { + DecodeResult { + output: DecodeOutput::Line(LogLine { + level, + text: "line".to_owned(), + }), + anomaly: None, + } + }) ); } @@ -414,7 +357,7 @@ mod tests { } #[test] - fn default_construction() { + fn a_default_decoder_decodes_like_a_new_one() { let mut default_decoder = LogDecoder::default(); let new_decoder_result = LogDecoder::new().feed(IncomingLogLevel::Info, "compare\n"); let default_result = default_decoder.feed(IncomingLogLevel::Info, "compare\n"); diff --git a/llama-cpp-log-decoder/src/log_level.rs b/llama-cpp-log-decoder/src/log_level.rs index 0fa157be0..3dc7d8a89 100644 --- a/llama-cpp-log-decoder/src/log_level.rs +++ b/llama-cpp-log-decoder/src/log_level.rs @@ -13,7 +13,7 @@ mod tests { use super::LogLevel; #[test] - fn unknown_variant_equality() { + fn unknown_levels_are_equal_only_with_the_same_raw_value() { assert_eq!(LogLevel::Unknown(42), LogLevel::Unknown(42)); assert_ne!(LogLevel::Unknown(42), LogLevel::Unknown(43)); } diff --git a/llama-cpp-test-harness-macros/src/attribute_accumulator.rs b/llama-cpp-test-harness-macros/src/attribute_accumulator.rs new file mode 100644 index 000000000..b7e677c54 --- /dev/null +++ b/llama-cpp-test-harness-macros/src/attribute_accumulator.rs @@ -0,0 +1,18 @@ +use crate::parsed_expected_failure::ParsedExpectedFailure; +use crate::parsed_load_mode::ParsedLoadMode; +use crate::parsed_source::ParsedSource; + +#[derive(Default)] +pub struct AttributeAccumulator { + pub model_source: Option, + pub mmproj_source: Option, + pub n_gpu_layers: Option, + pub load_mode: Option, + pub n_ctx: Option, + pub n_batch: Option, + pub n_ubatch: Option, + pub embeddings: Option, + pub n_seq_max: Option, + pub n_threads_batch: Option, + pub expected_failure: Option, +} diff --git a/llama-cpp-test-harness-macros/src/expand.rs b/llama-cpp-test-harness-macros/src/expand.rs index 787941c57..c5f48e297 100644 --- a/llama-cpp-test-harness-macros/src/expand.rs +++ b/llama-cpp-test-harness-macros/src/expand.rs @@ -1,15 +1,17 @@ -use proc_macro2::Ident; use proc_macro2::Span; use proc_macro2::TokenStream; use quote::quote; use syn::Item; use syn::ItemFn; use syn::ReturnType; +use syn::Signature; use syn::parse::Parse; use syn::parse::ParseStream; use syn::parse2; use crate::parsed_args::ParsedArgs; +use crate::parsed_expected_failure::ParsedExpectedFailure; +use crate::parsed_expected_outcome::ParsedExpectedOutcome; use crate::parsed_source::ParsedSource; struct StackedItems { @@ -33,12 +35,6 @@ fn validate_signature(item_fn: &ItemFn) -> syn::Result<()> { "llama_test functions must take exactly one argument: `&LlamaFixture<'_>`", )); } - if matches!(item_fn.sig.output, ReturnType::Default) { - return Err(syn::Error::new_spanned( - &item_fn.sig, - "llama_test functions must return `anyhow::Result<()>`", - )); - } if item_fn.sig.asyncness.is_some() { return Err(syn::Error::new_spanned( &item_fn.sig, @@ -79,13 +75,13 @@ fn split_fn_and_pass_through(items: Vec) -> syn::Result<(ItemFn, Vec fn build_model_source_literal(source: &ParsedSource) -> TokenStream { match source { ParsedSource::HuggingFace { repo, file } => quote! { - ::llama_cpp_test_harness::GgufSource::HuggingFace { + ::llama_cpp_test_model_provisioning::gguf_source::GgufSource::HuggingFace { repo: #repo, file: #file, } }, ParsedSource::LocalPath(path) => quote! { - ::llama_cpp_test_harness::GgufSource::LocalPath(#path) + ::llama_cpp_test_model_provisioning::gguf_source::GgufSource::LocalPath(#path) }, } } @@ -94,22 +90,93 @@ fn build_mmproj_source_literal(source: Option<&ParsedSource>) -> TokenStream { match source { None => quote! { ::core::option::Option::None }, Some(ParsedSource::HuggingFace { repo, file }) => quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::GgufSource::HuggingFace { + ::core::option::Option::Some(::llama_cpp_test_model_provisioning::gguf_source::GgufSource::HuggingFace { repo: #repo, file: #file, }) }, Some(ParsedSource::LocalPath(path)) => quote! { - ::core::option::Option::Some(::llama_cpp_test_harness::GgufSource::LocalPath(#path)) + ::core::option::Option::Some(::llama_cpp_test_model_provisioning::gguf_source::GgufSource::LocalPath(#path)) }, } } -fn build_registration(args: &ParsedArgs, fn_name: &Ident) -> TokenStream { - let trial_name = format!( - "{fn_name}[{suffix}]", - suffix = args.model_source.display_suffix() - ); +fn build_test_fn_literal(signature: &Signature) -> TokenStream { + let fn_name = &signature.ident; + + match signature.output { + ReturnType::Default => { + quote! { ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Infallible(#fn_name) } + } + ReturnType::Type(..) => { + quote! { ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Fallible(#fn_name) } + } + } +} + +fn validate_expected_outcome( + expected_outcome: &ParsedExpectedOutcome, + signature: &Signature, +) -> syn::Result<()> { + if let ( + ParsedExpectedOutcome::Failure(ParsedExpectedFailure::Test { .. }), + ReturnType::Default, + ) = (expected_outcome, &signature.output) + { + Err(syn::Error::new_spanned( + signature, + "`expected_failure = TestFailure(...)` needs a test that returns a `Result`, because \ + only an error it returns counts as its failure", + )) + } else { + Ok(()) + } +} + +fn build_expected_outcome_literal(expected_outcome: &ParsedExpectedOutcome) -> TokenStream { + match expected_outcome { + ParsedExpectedOutcome::Success => { + quote! { ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Success } + } + ParsedExpectedOutcome::Failure(ParsedExpectedFailure::PhaseSetup { kind, reason }) => { + quote! { + ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Failure { + failure: ::llama_cpp_test_harness::expected_failure::ExpectedFailure::PhaseSetup( + ::llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind::#kind + ), + reason: #reason, + } + } + } + ParsedExpectedOutcome::Failure(ParsedExpectedFailure::Test { reason }) => quote! { + ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Failure { + failure: ::llama_cpp_test_harness::expected_failure::ExpectedFailure::Test, + reason: #reason, + } + }, + } +} + +fn trial_name(args: &ParsedArgs, signature: &Signature) -> String { + let mmproj = args + .mmproj_source + .as_ref() + .map_or_else(String::new, |mmproj_source| { + format!(" mmproj={}", mmproj_source.display_suffix()) + }); + + format!( + "{fn_name}[{model}{mmproj} n_ctx={n_ctx} n_batch={n_batch} n_ubatch={n_ubatch}]", + fn_name = signature.ident, + model = args.model_source.display_suffix(), + n_ctx = args.context_params.n_ctx, + n_batch = args.context_params.n_batch, + n_ubatch = args.context_params.n_ubatch, + ) +} + +fn build_registration(args: &ParsedArgs, signature: &Signature) -> TokenStream { + let trial_name = trial_name(args, signature); let model_source_literal = build_model_source_literal(&args.model_source); let mmproj_source_literal = build_mmproj_source_literal(args.mmproj_source.as_ref()); let gpu_layers = args.model_load_params.n_gpu_layers; @@ -119,25 +186,26 @@ fn build_registration(args: &ParsedArgs, fn_name: &Ident) -> TokenStream { let physical_batch = args.context_params.n_ubatch; let embeddings_flag = args.context_params.embeddings; let sequence_max = args.context_params.n_seq_max; - let void_logs_flag = args.void_logs; + let test_fn = build_test_fn_literal(signature); + let expected_outcome = build_expected_outcome_literal(&args.expected_outcome); let threads_batch = args.context_params.n_threads_batch.map_or_else( || quote! { ::core::option::Option::None }, |value| quote! { ::core::option::Option::Some(#value) }, ); quote! { - ::llama_cpp_test_harness::inventory::submit! { - ::llama_cpp_test_harness::LlamaTestRegistration { + ::inventory::submit! { + ::llama_cpp_test_harness::llama_test_registration::LlamaTestRegistration { name: #trial_name, - key: ::llama_cpp_test_harness::LoadKey { + key: ::llama_cpp_test_model_provisioning::load_key::LoadKey { model_source: #model_source_literal, mmproj_source: #mmproj_source_literal, - model_load_params: ::llama_cpp_test_harness::ModelLoadParams { + model_load_params: ::llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams { n_gpu_layers: #gpu_layers, load_mode: #load_mode, }, }, - context_params: ::llama_cpp_test_harness::ContextParams { + context_params: ::llama_cpp_test_harness::context_params::ContextParams { n_ctx: #context_size, n_batch: #logical_batch, n_ubatch: #physical_batch, @@ -145,8 +213,8 @@ fn build_registration(args: &ParsedArgs, fn_name: &Ident) -> TokenStream { n_threads_batch: #threads_batch, embeddings: #embeddings_flag, }, - void_logs: #void_logs_flag, - func: #fn_name, + func: #test_fn, + expected_outcome: #expected_outcome, } } } @@ -157,9 +225,9 @@ pub fn expand(attribute: TokenStream, item: TokenStream) -> syn::Result syn::Result TokenStream { quote! { @@ -192,245 +275,324 @@ mod tests { } } - #[test] - fn well_formed_input_expands_to_function_plus_submission() { - let expanded = expand(well_formed_attribute(), well_formed_function()) - .expect("well-formed input must expand") - .to_string(); + fn expansion_error(attribute: TokenStream, item: TokenStream) -> String { + expand(attribute, item) + .expect_err("the input must be refused") + .to_string() + } - assert!( - expanded.contains("fn my_test"), - "expansion missing the original fn: {expanded}" - ); - assert!( - expanded.contains("LlamaTestRegistration"), - "expansion missing LlamaTestRegistration: {expanded}", - ); - assert!( - expanded.contains("\"my_test[bar.gguf]\""), - "expansion missing the trial-name literal with file suffix: {expanded}", - ); - assert!( - expanded.contains("GgufSource :: HuggingFace"), - "expansion missing GgufSource::HuggingFace variant: {expanded}", - ); - assert!( - expanded.contains("func : my_test"), - "expansion missing func wire-up: {expanded}", - ); + fn test_fn_literal_of(item: TokenStream) -> String { + let item_fn: ItemFn = parse2(item).expect("the item must be a function"); + + build_test_fn_literal(&item_fn.sig).to_string() } #[test] - fn expansion_with_local_path_model_source_emits_local_variant() { - let attribute = quote! { - model_source = LocalPath("/abs/local.gguf"), - n_gpu_layers = 0, - load_mode = Mmap, - n_ctx = 1, - n_batch = 1, - n_ubatch = 1 + fn expansion_keeps_the_function_and_submits_its_registration() { + let expanded = expand(well_formed_attribute(), well_formed_function()) + .expect("well-formed input must expand"); + let expected = quote! { + fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + ::inventory::submit! { + ::llama_cpp_test_harness::llama_test_registration::LlamaTestRegistration { + name: "my_test[bar.gguf n_ctx=1 n_batch=1 n_ubatch=1]", + key: ::llama_cpp_test_model_provisioning::load_key::LoadKey { + model_source: ::llama_cpp_test_model_provisioning::gguf_source::GgufSource::HuggingFace { + repo: "foo", + file: "bar.gguf", + }, + mmproj_source: ::core::option::Option::None, + model_load_params: ::llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams { + n_gpu_layers: 0i32, + load_mode: ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::Mmap, + }, + }, + context_params: ::llama_cpp_test_harness::context_params::ContextParams { + n_ctx: 1u32, + n_batch: 1u32, + n_ubatch: 1u32, + n_seq_max: 1u32, + n_threads_batch: ::core::option::Option::None, + embeddings: false, + }, + func: ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Fallible(my_test), + expected_outcome: ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Success, + } + } }; - let expanded = expand(attribute, well_formed_function()) - .expect("LocalPath must expand") - .to_string(); - assert!( - expanded.contains("GgufSource :: LocalPath"), - "expansion missing GgufSource::LocalPath variant: {expanded}", - ); - assert!( - expanded.contains("\"my_test[local.gguf]\""), - "trial name must use the path's filename component: {expanded}", - ); + assert_eq!(expanded.to_string(), expected.to_string()); } #[test] - fn expansion_with_mmproj_source_emits_some_variant() { - let attribute = quote! { - model_source = HuggingFace("r", "f"), - n_gpu_layers = 0, - load_mode = Mmap, - n_ctx = 1, - n_batch = 1, - n_ubatch = 1, - mmproj_source = HuggingFace("r", "mmproj.gguf") - }; - let expanded = expand(attribute, well_formed_function()) - .expect("mmproj_source must expand") - .to_string(); + fn a_setup_failure_expectation_emits_its_kind_and_reason() { + assert_eq!( + build_expected_outcome_literal(&ParsedExpectedOutcome::Failure( + ParsedExpectedFailure::PhaseSetup { + kind: Ident::new("ModelDownload", Span::call_site()), + reason: "the repository does not exist".to_owned(), + } + )) + .to_string(), + quote! { + ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Failure { + failure: ::llama_cpp_test_harness::expected_failure::ExpectedFailure::PhaseSetup( + ::llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind::ModelDownload + ), + reason: "the repository does not exist", + } + } + .to_string() + ); + } - assert!( - expanded.contains("GgufSource :: HuggingFace"), - "expansion missing GgufSource::HuggingFace: {expanded}", + #[test] + fn a_test_failure_expectation_emits_its_reason() { + assert_eq!( + build_expected_outcome_literal(&ParsedExpectedOutcome::Failure( + ParsedExpectedFailure::Test { + reason: "the body bails on purpose".to_owned(), + } + )) + .to_string(), + quote! { + ::llama_cpp_test_harness::expected_outcome::ExpectedOutcome::Failure { + failure: ::llama_cpp_test_harness::expected_failure::ExpectedFailure::Test, + reason: "the body bails on purpose", + } + } + .to_string() ); - assert!( - expanded.contains("Some"), - "expansion missing Option::Some wrap: {expanded}", + } + + #[test] + fn a_test_failure_expected_of_a_test_that_cannot_return_an_error_is_refused() { + assert_eq!( + expansion_error( + quote! { + model_source = HuggingFace("foo", "bar.gguf"), + n_gpu_layers = 0, + load_mode = Mmap, + n_ctx = 1, + n_batch = 1, + n_ubatch = 1, + expected_failure = TestFailure("the body bails on purpose") + }, + quote! { fn my_test(fixture: &LlamaFixture<'_>) {} } + ), + "`expected_failure = TestFailure(...)` needs a test that returns a `Result`, because \ + only an error it returns counts as its failure" ); } #[test] - fn expansion_with_local_path_mmproj_emits_local_variant() { - let attribute = quote! { - model_source = HuggingFace("r", "f"), + fn a_trial_with_an_mmproj_names_it_beside_its_model_and_context_sizes() { + let args: ParsedArgs = parse2(quote! { + model_source = HuggingFace("foo", "bar.gguf"), + mmproj_source = LocalPath("/models/mmproj.gguf"), n_gpu_layers = 0, load_mode = Mmap, - n_ctx = 1, - n_batch = 1, - n_ubatch = 1, - mmproj_source = LocalPath("/abs/mmproj.gguf") - }; - let expanded = expand(attribute, well_formed_function()) - .expect("LocalPath mmproj must expand") - .to_string(); - - assert!( - expanded.contains("GgufSource :: LocalPath"), - "expansion missing GgufSource::LocalPath: {expanded}", + n_ctx = 512, + n_batch = 128, + n_ubatch = 64 + }) + .expect("the attribute must parse"); + let item_fn: ItemFn = parse2(well_formed_function()).expect("the item must be a function"); + + assert_eq!( + trial_name(&args, &item_fn.sig), + "my_test[bar.gguf mmproj=mmproj.gguf n_ctx=512 n_batch=128 n_ubatch=64]" ); } #[test] - fn expansion_without_mmproj_emits_none_variant() { - let expanded = expand(well_formed_attribute(), well_formed_function()) - .expect("no-mmproj must expand") - .to_string(); - - assert!( - expanded.contains("None"), - "expansion missing Option::None for absent mmproj: {expanded}" + fn a_function_returning_a_result_registers_as_fallible() { + assert_eq!( + test_fn_literal_of(well_formed_function()), + quote! { ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Fallible(my_test) } + .to_string() ); } #[test] - fn malformed_attribute_propagates_parser_error() { - let attribute = quote! { totally_wrong = "x" }; - let error = expand(attribute, well_formed_function()) - .expect_err("malformed must fail") - .to_string(); - - assert!(error.contains("unknown field"), "got: {error}"); + fn a_function_without_a_return_type_registers_as_infallible() { + assert_eq!( + test_fn_literal_of(quote! { fn my_test(fixture: &LlamaFixture<'_>) {} }), + quote! { ::llama_cpp_test_harness::llama_test_fn::LlamaTestFn::Infallible(my_test) } + .to_string() + ); } #[test] - fn function_with_too_many_arguments_is_rejected() { - let item = quote! { - fn my_test(a: &LlamaFixture<'_>, b: i32) -> anyhow::Result<()> { Ok(()) } - }; - let error = expand(well_formed_attribute(), item) - .expect_err("two-arg must fail") - .to_string(); - - assert!(error.contains("exactly one argument"), "got: {error}"); + fn a_local_path_model_source_emits_the_local_path_variant() { + assert_eq!( + build_model_source_literal(&ParsedSource::LocalPath("/abs/local.gguf".to_owned())) + .to_string(), + quote! { ::llama_cpp_test_model_provisioning::gguf_source::GgufSource::LocalPath("/abs/local.gguf") } + .to_string() + ); } #[test] - fn function_with_zero_arguments_is_rejected() { - let item = quote! { fn my_test() -> anyhow::Result<()> { Ok(()) } }; - let error = expand(well_formed_attribute(), item) - .expect_err("zero-arg must fail") - .to_string(); - - assert!(error.contains("exactly one argument"), "got: {error}"); + fn a_huggingface_mmproj_source_is_wrapped_in_some() { + assert_eq!( + build_mmproj_source_literal(Some(&ParsedSource::HuggingFace { + repo: "r".to_owned(), + file: "mmproj.gguf".to_owned(), + })) + .to_string(), + quote! { + ::core::option::Option::Some(::llama_cpp_test_model_provisioning::gguf_source::GgufSource::HuggingFace { + repo: "r", + file: "mmproj.gguf", + }) + } + .to_string() + ); } #[test] - fn impl_block_input_is_rejected_at_parse_time() { - let item = quote! { - impl S { - fn my_test(&self) -> anyhow::Result<()> { Ok(()) } + fn a_local_path_mmproj_source_is_wrapped_in_some() { + assert_eq!( + build_mmproj_source_literal(Some(&ParsedSource::LocalPath( + "/abs/mmproj.gguf".to_owned() + ))) + .to_string(), + quote! { + ::core::option::Option::Some(::llama_cpp_test_model_provisioning::gguf_source::GgufSource::LocalPath( + "/abs/mmproj.gguf" + )) } - }; - let result = expand(well_formed_attribute(), item); - - assert!(result.is_err(), "impl block must fail"); + .to_string() + ); } #[test] - fn function_without_return_type_is_rejected() { - let item = quote! { fn my_test(fixture: &LlamaFixture<'_>) { } }; - let error = expand(well_formed_attribute(), item) - .expect_err("missing return type must fail") - .to_string(); - - assert!(error.contains("anyhow::Result"), "got: {error}"); + fn a_malformed_attribute_propagates_the_parser_error() { + assert_eq!( + expansion_error(quote! { totally_wrong = "x" }, well_formed_function()), + "unknown field `totally_wrong`; expected one of: model_source, n_gpu_layers, load_mode, \ + n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch, \ + expected_failure" + ); } #[test] - fn async_function_is_rejected() { - let item = quote! { - async fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } - }; - let error = expand(well_formed_attribute(), item) - .expect_err("async must fail") - .to_string(); - - assert!(error.contains("synchronous"), "got: {error}"); + fn a_function_with_two_arguments_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { fn my_test(a: &LlamaFixture<'_>, b: i32) -> anyhow::Result<()> { Ok(()) } }, + ), + "llama_test functions must take exactly one argument: `&LlamaFixture<'_>`" + ); } #[test] - fn generic_function_is_rejected() { - let item = quote! { - fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } - }; - let error = expand(well_formed_attribute(), item) - .expect_err("generic must fail") - .to_string(); - - assert!(error.contains("generic"), "got: {error}"); + fn a_function_without_arguments_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { fn my_test() -> anyhow::Result<()> { Ok(()) } }, + ), + "llama_test functions must take exactly one argument: `&LlamaFixture<'_>`" + ); } #[test] - fn malformed_item_token_stream_is_rejected() { - let item = quote! { this is not a function }; - let result = expand(well_formed_attribute(), item); - - assert!(result.is_err(), "non-fn item must fail"); + fn an_async_function_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { + async fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + }, + ), + "llama_test functions must be synchronous" + ); } #[test] - fn stacked_invocation_preserves_prior_submission() { - let prior_layer_output = expand(well_formed_attribute(), well_formed_function()) - .expect("first layer must expand"); - - let second_attribute = quote! { - model_source = HuggingFace("second", "second.gguf"), - n_gpu_layers = 1, - load_mode = None, - n_ctx = 2, - n_batch = 2, - n_ubatch = 2 - }; - - let second_layer_output = expand(second_attribute, prior_layer_output) - .expect("stacked second invocation must expand") - .to_string(); - - assert!( - second_layer_output.contains("\"my_test[bar.gguf]\""), - "stacked output missing first trial name: {second_layer_output}", + fn a_generic_function_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { + fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + }, + ), + "llama_test functions must not be generic" ); - assert!( - second_layer_output.contains("\"my_test[second.gguf]\""), - "stacked output missing second trial name: {second_layer_output}", + } + + #[test] + fn an_impl_block_is_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { + impl S { + fn my_test(&self) -> anyhow::Result<()> { Ok(()) } + } + }, + ), + "llama_test expects an fn definition" ); - let occurrences = second_layer_output.matches("LlamaTestRegistration").count(); - assert!( - occurrences >= 2, - "stacked output should contain two LlamaTestRegistration submissions, found {occurrences}: {second_layer_output}", + } + + #[test] + fn tokens_that_are_not_an_item_are_refused() { + assert_eq!( + expansion_error(well_formed_attribute(), quote! { this is not a function }), + "expected `!`" ); } #[test] - fn two_fn_definitions_in_input_are_rejected() { - let item = quote! { - fn first(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } - fn second(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } - }; - let error = expand(well_formed_attribute(), item) - .expect_err("two fns must fail") - .to_string(); + fn two_function_definitions_are_refused() { + assert_eq!( + expansion_error( + well_formed_attribute(), + quote! { + fn first(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + fn second(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } + }, + ), + "llama_test expects exactly one fn definition" + ); + } - assert!(error.contains("exactly one fn"), "got: {error}"); + #[test] + fn a_stacked_invocation_keeps_the_registration_of_the_previous_layer() { + let first_layer = expand(well_formed_attribute(), well_formed_function()) + .expect("the first layer must expand"); + let second_layer = expand( + quote! { + model_source = HuggingFace("second", "second.gguf"), + n_gpu_layers = 1, + load_mode = None, + n_ctx = 2, + n_batch = 2, + n_ubatch = 2 + }, + first_layer, + ) + .expect("the stacked layer must expand"); + let StackedItems { items } = parse2(second_layer).expect("the expansion must be items"); + + assert_eq!( + items + .iter() + .map(|item| matches!(item, Item::Fn(_))) + .collect::>(), + [true, false, false] + ); + assert_eq!( + items + .iter() + .map(|item| matches!(item, Item::Macro(_))) + .collect::>(), + [false, true, true] + ); } } diff --git a/llama-cpp-test-harness-macros/src/lib.rs b/llama-cpp-test-harness-macros/src/lib.rs index 04b38d8fb..c58d222f0 100644 --- a/llama-cpp-test-harness-macros/src/lib.rs +++ b/llama-cpp-test-harness-macros/src/lib.rs @@ -3,9 +3,12 @@ deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) )] +mod attribute_accumulator; mod expand; mod parsed_args; mod parsed_context_params; +mod parsed_expected_failure; +mod parsed_expected_outcome; mod parsed_load_mode; mod parsed_model_load_params; mod parsed_source; @@ -32,6 +35,7 @@ mod tests { use quote::quote; use super::dispatch; + use crate::expand::expand; #[test] fn dispatch_on_invalid_attribute_emits_compile_error_tokens() { @@ -39,16 +43,18 @@ mod tests { let item = quote! { fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } }; - let emitted = dispatch(attribute, item).to_string(); + let message = "unknown field `totally_wrong`; expected one of: model_source, \ + n_gpu_layers, load_mode, n_ctx, n_batch, n_ubatch, mmproj_source, \ + embeddings, n_seq_max, n_threads_batch, expected_failure"; - assert!( - emitted.contains("compile_error"), - "expected compile_error! tokens in emitted output: {emitted}", + assert_eq!( + dispatch(attribute, item).to_string(), + quote! { ::core::compile_error! { #message } }.to_string() ); } #[test] - fn dispatch_on_valid_input_emits_inventory_submission() { + fn dispatch_on_valid_input_emits_the_expanded_tokens() { let attribute = quote! { model_source = HuggingFace("r", "f"), n_gpu_layers = 0, @@ -60,15 +66,8 @@ mod tests { let item = quote! { fn my_test(fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { Ok(()) } }; - let emitted = dispatch(attribute, item).to_string(); + let expanded = expand(attribute.clone(), item.clone()).expect("valid input must expand"); - assert!( - emitted.contains("inventory"), - "expected inventory::submit in emitted output: {emitted}", - ); - assert!( - !emitted.contains("compile_error"), - "valid input should not emit compile_error: {emitted}", - ); + assert_eq!(dispatch(attribute, item).to_string(), expanded.to_string()); } } diff --git a/llama-cpp-test-harness-macros/src/parsed_args.rs b/llama-cpp-test-harness-macros/src/parsed_args.rs index e262e64cc..705b4d005 100644 --- a/llama-cpp-test-harness-macros/src/parsed_args.rs +++ b/llama-cpp-test-harness-macros/src/parsed_args.rs @@ -4,14 +4,19 @@ use proc_macro2::Ident; use proc_macro2::Span; use syn::Expr; use syn::ExprLit; +use syn::ExprUnary; use syn::Lit; use syn::Meta; use syn::Token; +use syn::UnOp; use syn::parse::Parse; use syn::parse::ParseStream; use syn::punctuated::Punctuated; +use crate::attribute_accumulator::AttributeAccumulator; use crate::parsed_context_params::ParsedContextParams; +use crate::parsed_expected_failure::ParsedExpectedFailure; +use crate::parsed_expected_outcome::ParsedExpectedOutcome; use crate::parsed_load_mode::ParsedLoadMode; use crate::parsed_model_load_params::ParsedModelLoadParams; use crate::parsed_source::ParsedSource; @@ -30,7 +35,7 @@ const OPTIONAL_FIELDS: &[&str] = &[ "embeddings", "n_seq_max", "n_threads_batch", - "void_logs", + "expected_failure", ]; fn literal_from_expression(expression: &Expr) -> syn::Result<&Lit> { @@ -78,6 +83,41 @@ fn require_i32_lit(literal: &Lit, field: &str) -> syn::Result { } } +fn require_negated_i32_lit(literal: &Lit, field: &str) -> syn::Result { + let Lit::Int(int_literal) = literal else { + return Err(syn::Error::new_spanned( + literal, + format!("field `{field}` expects an integer literal"), + )); + }; + let magnitude = int_literal.base10_parse::().map_err(|parse_error| { + syn::Error::new_spanned( + literal, + format!("field `{field}` expects an integer that fits in i32: {parse_error}"), + ) + })?; + + 0_i32.checked_sub_unsigned(magnitude).ok_or_else(|| { + syn::Error::new_spanned( + literal, + format!("field `{field}` expects an integer that fits in i32: -{magnitude} is below its minimum"), + ) + }) +} + +fn require_i32_value(expression: &Expr, field: &str) -> syn::Result { + if let Expr::Unary(ExprUnary { + op: UnOp::Neg(_), + expr: negated, + .. + }) = expression + { + require_negated_i32_lit(literal_from_expression(negated)?, field) + } else { + require_i32_lit(literal_from_expression(expression)?, field) + } +} + fn require_bool_lit(literal: &Lit, field: &str) -> syn::Result { if let Lit::Bool(bool_literal) = literal { Ok(bool_literal.value()) @@ -93,21 +133,6 @@ fn require(value: Option, field: &str, span: Span) -> syn::Resul value.ok_or_else(|| syn::Error::new(span, format!("missing required field `{field}`"))) } -#[derive(Default)] -struct AttributeAccumulator { - model_source: Option, - mmproj_source: Option, - n_gpu_layers: Option, - load_mode: Option, - n_ctx: Option, - n_batch: Option, - n_ubatch: Option, - embeddings: Option, - n_seq_max: Option, - n_threads_batch: Option, - void_logs: Option, -} - fn dispatch_field( accumulator: &mut AttributeAccumulator, identifier: &Ident, @@ -122,10 +147,7 @@ fn dispatch_field( accumulator.mmproj_source = Some(ParsedSource::parse(value, "mmproj_source")?); } "n_gpu_layers" => { - accumulator.n_gpu_layers = Some(require_i32_lit( - literal_from_expression(value)?, - "n_gpu_layers", - )?); + accumulator.n_gpu_layers = Some(require_i32_value(value, "n_gpu_layers")?); } "n_ctx" => { accumulator.n_ctx = Some(require_int_lit(literal_from_expression(value)?, "n_ctx")?); @@ -168,16 +190,10 @@ fn dispatch_field( )?); } "n_threads_batch" => { - accumulator.n_threads_batch = Some(require_i32_lit( - literal_from_expression(value)?, - "n_threads_batch", - )?); + accumulator.n_threads_batch = Some(require_i32_value(value, "n_threads_batch")?); } - "void_logs" => { - accumulator.void_logs = Some(require_bool_lit( - literal_from_expression(value)?, - "void_logs", - )?); + "expected_failure" => { + accumulator.expected_failure = Some(ParsedExpectedFailure::parse(value)?); } "repo" | "file" | "mmproj_file" => { return Err(syn::Error::new_spanned( @@ -207,7 +223,7 @@ pub struct ParsedArgs { pub mmproj_source: Option, pub model_load_params: ParsedModelLoadParams, pub context_params: ParsedContextParams, - pub void_logs: bool, + pub expected_outcome: ParsedExpectedOutcome, } impl Parse for ParsedArgs { @@ -253,7 +269,10 @@ impl Parse for ParsedArgs { n_threads_batch: accumulator.n_threads_batch, embeddings: accumulator.embeddings.unwrap_or(false), }, - void_logs: accumulator.void_logs.unwrap_or(false), + expected_outcome: accumulator.expected_failure.map_or( + ParsedExpectedOutcome::Success, + ParsedExpectedOutcome::Failure, + ), }) } } @@ -263,6 +282,7 @@ mod tests { use syn::parse_str; use super::ParsedArgs; + use crate::parsed_load_mode::ParsedLoadMode; use crate::parsed_source::ParsedSource; const ALL_REQUIRED: &str = "\ @@ -289,10 +309,7 @@ mod tests { }, ); assert_eq!(parsed.model_load_params.n_gpu_layers, 0); - assert_eq!( - parsed.model_load_params.load_mode, - crate::parsed_load_mode::ParsedLoadMode::Mmap - ); + assert_eq!(parsed.model_load_params.load_mode, ParsedLoadMode::Mmap); assert_eq!(parsed.context_params.n_ctx, 512); assert_eq!(parsed.context_params.n_batch, 128); assert_eq!(parsed.context_params.n_ubatch, 64); @@ -350,10 +367,7 @@ mod tests { .expect_err("missing model_source must fail") .to_string(); - assert!( - message.contains("missing required field `model_source`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `model_source`"); } #[test] @@ -363,10 +377,7 @@ mod tests { .expect_err("missing n_ctx must fail") .to_string(); - assert!( - message.contains("missing required field `n_ctx`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `n_ctx`"); } #[test] @@ -377,7 +388,11 @@ mod tests { .expect_err("legacy repo must be rejected") .to_string(); - assert!(message.contains("model_source"), "got: {message}"); + assert_eq!( + message, + "field `repo` was removed; use `model_source = HuggingFace(repo, file)` or \ + `model_source = LocalPath(path)` (and `mmproj_source` for mmproj)" + ); } #[test] @@ -387,7 +402,24 @@ mod tests { .expect_err("legacy mmproj_file must be rejected") .to_string(); - assert!(message.contains("mmproj_source"), "got: {message}"); + assert_eq!( + message, + "field `mmproj_file` was removed; use `model_source = HuggingFace(repo, file)` or \ + `model_source = LocalPath(path)` (and `mmproj_source` for mmproj)" + ); + } + + #[test] + fn an_expected_failure_that_cannot_be_parsed_is_rejected() { + let source = format!("{ALL_REQUIRED}, expected_failure = Whatever(\"reason\")"); + + assert_eq!( + parse(&source) + .expect_err("an unknown expected failure must fail") + .to_string(), + "field `expected_failure` expects `SetupFailure(kind, \"reason\")` or \ + `TestFailure(\"reason\")`" + ); } #[test] @@ -397,9 +429,11 @@ mod tests { .expect_err("unknown field must fail") .to_string(); - assert!( - message.contains("unknown field `surprise`"), - "got: {message}" + assert_eq!( + message, + "unknown field `surprise`; expected one of: model_source, n_gpu_layers, load_mode, \ + n_ctx, n_batch, n_ubatch, mmproj_source, embeddings, n_seq_max, n_threads_batch, \ + expected_failure" ); } @@ -408,10 +442,7 @@ mod tests { let source = format!("{ALL_REQUIRED}, model_source = HuggingFace(\"other\", \"o.gguf\")"); let message = parse(&source).expect_err("duplicate must fail").to_string(); - assert!( - message.contains("duplicate field `model_source`"), - "got: {message}" - ); + assert_eq!(message, "duplicate field `model_source`"); } #[test] @@ -419,75 +450,139 @@ mod tests { let source = "model_source, file = \"x\""; let message = parse(source).expect_err("bare ident must fail").to_string(); - assert!(message.contains("name = value"), "got: {message}"); + assert_eq!(message, "expected `name = value` form"); } #[test] - fn non_literal_value_for_scalar_field_is_rejected() { + fn load_mode_literal_is_rejected() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ - n_gpu_layers = some_const, \ - load_mode = Mmap, \ + n_gpu_layers = 0, \ + load_mode = \"Mmap\", \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; let message = parse(source) - .expect_err("non-literal value must fail") + .expect_err("literal load mode must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a load-mode identifier"); } #[test] - fn wrong_literal_kind_for_int_field_is_rejected() { + fn qualified_load_mode_path_is_rejected() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ - n_gpu_layers = \"nine\", \ - load_mode = Mmap, \ + n_gpu_layers = 0, \ + load_mode = modes::Mmap, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; let message = parse(source) - .expect_err("string for int field must fail") + .expect_err("qualified load mode must fail") .to_string(); - assert!(message.contains("integer literal"), "got: {message}"); + assert_eq!(message, "expected a load-mode identifier"); } #[test] - fn load_mode_literal_is_rejected() { + fn negative_i32_fields_are_parsed() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ - n_gpu_layers = 0, \ - load_mode = \"Mmap\", \ + n_gpu_layers = -1, \ + load_mode = Mmap, \ + n_ctx = 1, \ + n_batch = 1, \ + n_ubatch = 1, \ + n_threads_batch = -2"; + let parsed = parse(source).expect("negative i32 fields must parse"); + + assert_eq!( + [ + parsed.model_load_params.n_gpu_layers, + parsed + .context_params + .n_threads_batch + .expect("n_threads_batch was given") + ], + [-1, -2] + ); + } + + #[test] + fn a_negative_i32_field_below_i32_min_is_rejected() { + let source = "\ + model_source = HuggingFace(\"x\", \"y\"), \ + n_gpu_layers = -2147483649, \ + load_mode = Mmap, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; - let message = parse(source) - .expect_err("literal load mode must fail") - .to_string(); - assert!(message.contains("load-mode identifier"), "got: {message}"); + assert_eq!( + parse(source) + .expect_err("an i32 below i32::MIN must fail") + .to_string(), + "field `n_gpu_layers` expects an integer that fits in i32: -2147483649 is below its minimum" + ); } #[test] - fn qualified_load_mode_path_is_rejected() { + fn a_negative_i32_field_whose_magnitude_exceeds_u32_is_rejected() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ - n_gpu_layers = 0, \ - load_mode = modes::Mmap, \ + n_gpu_layers = -4294967296, \ + load_mode = Mmap, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; - let message = parse(source) - .expect_err("qualified load mode must fail") - .to_string(); - assert!(message.contains("load-mode identifier"), "got: {message}"); + assert_eq!( + parse(source) + .expect_err("a magnitude beyond u32 must fail") + .to_string(), + "field `n_gpu_layers` expects an integer that fits in i32: number too large to fit in target type" + ); + } + + #[test] + fn a_negated_i32_field_that_is_not_an_integer_is_rejected() { + let source = "\ + model_source = HuggingFace(\"x\", \"y\"), \ + n_gpu_layers = -1.5, \ + load_mode = Mmap, \ + n_ctx = 1, \ + n_batch = 1, \ + n_ubatch = 1"; + + assert_eq!( + parse(source) + .expect_err("a negated float must fail") + .to_string(), + "field `n_gpu_layers` expects an integer literal" + ); + } + + #[test] + fn a_negated_i32_field_that_is_not_a_literal_is_rejected() { + let source = "\ + model_source = HuggingFace(\"x\", \"y\"), \ + n_gpu_layers = -layers, \ + load_mode = Mmap, \ + n_ctx = 1, \ + n_batch = 1, \ + n_ubatch = 1"; + + assert_eq!( + parse(source) + .expect_err("a negated identifier must fail") + .to_string(), + "expected a literal (string, integer, or bool)" + ); } #[test] - fn negative_int_for_u32_field_is_rejected() { + fn negative_int_for_u32_field_is_rejected_as_a_non_literal() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = 0, \ @@ -499,7 +594,7 @@ mod tests { .expect_err("negative int must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -516,7 +611,7 @@ mod tests { .expect_err("path field name must fail") .to_string(); - assert!(message.contains("simple identifier"), "got: {message}"); + assert_eq!(message, "expected a simple identifier"); } #[test] @@ -530,7 +625,11 @@ mod tests { n_ubatch = 1"; let message = parse(source).expect_err("overflow must fail").to_string(); - assert!(message.contains("u32"), "got: {message}"); + assert_eq!( + message, + "field `n_ctx` expects a non-negative integer that fits in u32: \ + number too large to fit in target type" + ); } #[test] @@ -540,7 +639,11 @@ mod tests { .expect_err("i32 overflow must fail") .to_string(); - assert!(message.contains("i32"), "got: {message}"); + assert_eq!( + message, + "field `n_threads_batch` expects an integer that fits in i32: \ + number too large to fit in target type" + ); } #[test] @@ -555,10 +658,7 @@ mod tests { .expect_err("missing n_gpu_layers must fail") .to_string(); - assert!( - message.contains("missing required field `n_gpu_layers`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `n_gpu_layers`"); } #[test] @@ -573,10 +673,7 @@ mod tests { .expect_err("missing load_mode must fail") .to_string(); - assert!( - message.contains("missing required field `load_mode`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `load_mode`"); } #[test] @@ -592,7 +689,10 @@ mod tests { .expect_err("unknown load mode must fail") .to_string(); - assert!(message.contains("MmapMlock"), "got: {message}"); + assert_eq!( + message, + "expected one of: Auto, None, Mmap, Mlock, MmapMlock, DirectIo" + ); } #[test] @@ -607,10 +707,7 @@ mod tests { .expect_err("missing n_batch must fail") .to_string(); - assert!( - message.contains("missing required field `n_batch`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `n_batch`"); } #[test] @@ -625,10 +722,7 @@ mod tests { .expect_err("missing n_ubatch must fail") .to_string(); - assert!( - message.contains("missing required field `n_ubatch`"), - "got: {message}" - ); + assert_eq!(message, "missing required field `n_ubatch`"); } #[test] @@ -646,14 +740,6 @@ mod tests { assert!(parsed.context_params.embeddings); } - #[test] - fn optional_embeddings_false_is_parsed() { - let source = format!("{ALL_REQUIRED}, embeddings = false"); - let parsed = parse(&source).expect("embeddings = false must parse"); - - assert!(!parsed.context_params.embeddings); - } - #[test] fn optional_embeddings_rejects_non_bool_literal() { let source = format!("{ALL_REQUIRED}, embeddings = 1"); @@ -661,7 +747,10 @@ mod tests { .expect_err("embeddings with int must fail") .to_string(); - assert!(message.contains("bool literal"), "got: {message}"); + assert_eq!( + message, + "field `embeddings` expects a bool literal (`true` or `false`)" + ); } #[test] @@ -701,22 +790,10 @@ mod tests { .expect_err("string for n_threads_batch must fail") .to_string(); - assert!(message.contains("integer literal"), "got: {message}"); - } - - #[test] - fn optional_void_logs_defaults_to_false_when_absent() { - let parsed = parse(ALL_REQUIRED).expect("required-only must parse"); - - assert!(!parsed.void_logs); - } - - #[test] - fn optional_void_logs_true_is_parsed() { - let source = format!("{ALL_REQUIRED}, void_logs = true"); - let parsed = parse(&source).expect("void_logs = true must parse"); - - assert!(parsed.void_logs); + assert_eq!( + message, + "field `n_threads_batch` expects an integer literal" + ); } fn override_field(field: &str, replacement: &str) -> String { @@ -748,7 +825,10 @@ mod tests { let source = override_field(field, "some_const"); let message = parse(&source).expect_err(field).to_string(); - assert!(message.contains("literal"), "{field}: {message}"); + assert_eq!( + message, "expected a literal (string, integer, or bool)", + "{field}" + ); } } @@ -758,7 +838,10 @@ mod tests { let source = override_field(field, "\"not-an-int\""); let message = parse(&source).expect_err(field).to_string(); - assert!(message.contains("integer literal"), "{field}: {message}"); + assert_eq!( + message, + format!("field `{field}` expects an integer literal") + ); } } @@ -769,7 +852,7 @@ mod tests { .expect_err("n_seq_max non-literal must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -779,7 +862,7 @@ mod tests { .expect_err("n_seq_max wrong-kind must fail") .to_string(); - assert!(message.contains("integer literal"), "got: {message}"); + assert_eq!(message, "field `n_seq_max` expects an integer literal"); } #[test] @@ -789,7 +872,7 @@ mod tests { .expect_err("n_threads_batch non-literal must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -799,27 +882,7 @@ mod tests { .expect_err("embeddings non-literal must fail") .to_string(); - assert!(message.contains("literal"), "got: {message}"); - } - - #[test] - fn optional_void_logs_rejects_non_literal_value() { - let source = append_field("void_logs", "some_const"); - let message = parse(&source) - .expect_err("void_logs non-literal must fail") - .to_string(); - - assert!(message.contains("literal"), "got: {message}"); - } - - #[test] - fn optional_void_logs_rejects_wrong_literal_kind() { - let source = append_field("void_logs", "1"); - let message = parse(&source) - .expect_err("void_logs wrong-kind must fail") - .to_string(); - - assert!(message.contains("bool literal"), "got: {message}"); + assert_eq!(message, "expected a literal (string, integer, or bool)"); } #[test] @@ -829,7 +892,10 @@ mod tests { .expect_err("mmproj_source unknown variant must fail") .to_string(); - assert!(message.contains("unknown source variant"), "got: {message}"); + assert_eq!( + message, + "unknown source variant `Mystery`; expected `HuggingFace` or `LocalPath`" + ); } #[test] @@ -845,16 +911,18 @@ mod tests { .expect_err("model_source unknown variant must fail") .to_string(); - assert!(message.contains("unknown source variant"), "got: {message}"); + assert_eq!( + message, + "unknown source variant `Mystery`; expected `HuggingFace` or `LocalPath`" + ); } #[test] fn unparseable_attribute_token_stream_is_rejected() { - let result = parse("@&^!"); + let message = parse("@&^!") + .expect_err("garbage attribute body must fail") + .to_string(); - assert!( - result.is_err(), - "garbage attribute body must fail to parse as Punctuated" - ); + assert_eq!(message, "expected identifier"); } } diff --git a/llama-cpp-test-harness-macros/src/parsed_expected_failure.rs b/llama-cpp-test-harness-macros/src/parsed_expected_failure.rs new file mode 100644 index 000000000..250a88869 --- /dev/null +++ b/llama-cpp-test-harness-macros/src/parsed_expected_failure.rs @@ -0,0 +1,145 @@ +use proc_macro2::Ident; +use syn::Expr; +use syn::ExprCall; +use syn::ExprLit; +use syn::ExprPath; +use syn::Lit; + +const USAGE: &str = "field `expected_failure` expects `SetupFailure(kind, \"reason\")` or \ + `TestFailure(\"reason\")`"; + +fn reason_of(expression: &Expr) -> syn::Result { + if let Expr::Lit(ExprLit { + lit: Lit::Str(reason), + .. + }) = expression + && !reason.value().is_empty() + { + Ok(reason.value()) + } else { + Err(syn::Error::new_spanned( + expression, + "`expected_failure` expects a non-empty string stating why the trial fails on purpose", + )) + } +} + +fn setup_failure_kind_of(expression: &Expr) -> syn::Result { + if let Expr::Path(ExprPath { path, .. }) = expression + && let Some(kind) = path.get_ident() + { + Ok(kind.clone()) + } else { + Err(syn::Error::new_spanned( + expression, + "`SetupFailure` expects the bare name of a phase setup failure kind, such as \ + `ModelDownload`", + )) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ParsedExpectedFailure { + PhaseSetup { kind: Ident, reason: String }, + Test { reason: String }, +} + +impl ParsedExpectedFailure { + pub fn parse(expression: &Expr) -> syn::Result { + let Expr::Call(ExprCall { func, args, .. }) = expression else { + return Err(syn::Error::new_spanned(expression, USAGE)); + }; + let arguments: Vec<&Expr> = args.iter().collect(); + + match (func.as_ref(), arguments.as_slice()) { + (Expr::Path(ExprPath { path, .. }), [kind, reason]) + if path.is_ident("SetupFailure") => + { + Ok(Self::PhaseSetup { + kind: setup_failure_kind_of(kind)?, + reason: reason_of(reason)?, + }) + } + (Expr::Path(ExprPath { path, .. }), [reason]) if path.is_ident("TestFailure") => { + Ok(Self::Test { + reason: reason_of(reason)?, + }) + } + _ => Err(syn::Error::new_spanned(expression, USAGE)), + } + } +} + +#[cfg(test)] +mod tests { + use proc_macro2::Ident; + use proc_macro2::Span; + use syn::parse_str; + + use super::ParsedExpectedFailure; + + fn parse(source: &str) -> Result { + ParsedExpectedFailure::parse(&parse_str(source).expect("the source must be an expression")) + .map_err(|error| error.to_string()) + } + + #[test] + fn a_setup_failure_names_its_kind_and_reason() { + assert_eq!( + parse("SetupFailure(ModelDownload, \"the repository does not exist\")"), + Ok(ParsedExpectedFailure::PhaseSetup { + kind: Ident::new("ModelDownload", Span::call_site()), + reason: "the repository does not exist".to_owned(), + }) + ); + } + + #[test] + fn a_test_failure_names_its_reason() { + assert_eq!( + parse("TestFailure(\"the body bails on purpose\")"), + Ok(ParsedExpectedFailure::Test { + reason: "the body bails on purpose".to_owned(), + }) + ); + } + + #[test] + fn a_failure_that_is_not_a_call_is_refused() { + assert_eq!( + parse("ModelDownload"), + Err( + "field `expected_failure` expects `SetupFailure(kind, \"reason\")` or \ + `TestFailure(\"reason\")`" + .to_owned() + ) + ); + } + + #[test] + fn an_empty_reason_is_refused() { + assert_eq!( + [ + parse("SetupFailure(ModelDownload, \"\")"), + parse("TestFailure(\"\")") + ], + [(); 2].map(|()| Err( + "`expected_failure` expects a non-empty string stating why the trial fails on \ + purpose" + .to_owned() + )) + ); + } + + #[test] + fn a_setup_failure_kind_that_is_not_a_bare_name_is_refused() { + assert_eq!( + parse("SetupFailure(kinds::ModelDownload, \"reason\")"), + Err( + "`SetupFailure` expects the bare name of a phase setup failure kind, such as \ + `ModelDownload`" + .to_owned() + ) + ); + } +} diff --git a/llama-cpp-test-harness-macros/src/parsed_expected_outcome.rs b/llama-cpp-test-harness-macros/src/parsed_expected_outcome.rs new file mode 100644 index 000000000..e8b42d7c0 --- /dev/null +++ b/llama-cpp-test-harness-macros/src/parsed_expected_outcome.rs @@ -0,0 +1,7 @@ +use crate::parsed_expected_failure::ParsedExpectedFailure; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ParsedExpectedOutcome { + Success, + Failure(ParsedExpectedFailure), +} diff --git a/llama-cpp-test-harness-macros/src/parsed_load_mode.rs b/llama-cpp-test-harness-macros/src/parsed_load_mode.rs index 1f9773ffc..cf9a85460 100644 --- a/llama-cpp-test-harness-macros/src/parsed_load_mode.rs +++ b/llama-cpp-test-harness-macros/src/parsed_load_mode.rs @@ -30,12 +30,24 @@ impl ParsedLoadMode { pub fn tokens(self) -> TokenStream { match self { - Self::Auto => quote! { ::llama_cpp_test_harness::LlamaLoadMode::Auto }, - Self::None => quote! { ::llama_cpp_test_harness::LlamaLoadMode::None }, - Self::Mmap => quote! { ::llama_cpp_test_harness::LlamaLoadMode::Mmap }, - Self::Mlock => quote! { ::llama_cpp_test_harness::LlamaLoadMode::Mlock }, - Self::MmapMlock => quote! { ::llama_cpp_test_harness::LlamaLoadMode::MmapMlock }, - Self::DirectIo => quote! { ::llama_cpp_test_harness::LlamaLoadMode::DirectIo }, + Self::Auto => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::Auto } + } + Self::None => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::None } + } + Self::Mmap => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::Mmap } + } + Self::Mlock => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::Mlock } + } + Self::MmapMlock => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::MmapMlock } + } + Self::DirectIo => { + quote! { ::llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode::DirectIo } + } } } } @@ -65,7 +77,9 @@ mod tests { assert_eq!(parsed, expected); assert_eq!( parsed.tokens().to_string(), - format!(":: llama_cpp_test_harness :: LlamaLoadMode :: {identifier}"), + format!( + ":: llama_cpp_bindings :: model :: llama_load_mode :: LlamaLoadMode :: {identifier}" + ), ); } } @@ -77,8 +91,9 @@ mod tests { .expect_err("unknown mode must fail") .to_string(); - for valid in ["Auto", "None", "Mmap", "Mlock", "MmapMlock", "DirectIo"] { - assert!(message.contains(valid), "missing {valid} in {message}"); - } + assert_eq!( + message, + "expected one of: Auto, None, Mmap, Mlock, MmapMlock, DirectIo" + ); } } diff --git a/llama-cpp-test-harness-macros/src/parsed_source.rs b/llama-cpp-test-harness-macros/src/parsed_source.rs index bba3fa031..e2a07e308 100644 --- a/llama-cpp-test-harness-macros/src/parsed_source.rs +++ b/llama-cpp-test-harness-macros/src/parsed_source.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use syn::Expr; use syn::ExprCall; use syn::ExprLit; @@ -58,7 +60,7 @@ impl ParsedSource { pub fn display_suffix(&self) -> String { match self { Self::HuggingFace { file, .. } => file.clone(), - Self::LocalPath(path) => std::path::Path::new(path) + Self::LocalPath(path) => Path::new(path) .file_name() .and_then(|name| name.to_str()) .map_or_else(|| path.clone(), str::to_owned), @@ -104,7 +106,8 @@ mod tests { use super::ParsedSource; fn parse(source: &str) -> syn::Result { - let expression: syn::Expr = parse_str(source)?; + let expression: syn::Expr = + parse_str(source).expect("the test source must be an expression"); ParsedSource::parse(&expression, "model_source") } @@ -137,7 +140,10 @@ mod tests { .expect_err("unknown variant must fail") .to_string(); - assert!(message.contains("unknown source variant"), "got: {message}"); + assert_eq!( + message, + "unknown source variant `Mystery`; expected `HuggingFace` or `LocalPath`" + ); } #[test] @@ -146,8 +152,10 @@ mod tests { .expect_err("non-call must fail") .to_string(); - assert!(message.contains("HuggingFace"), "got: {message}"); - assert!(message.contains("LocalPath"), "got: {message}"); + assert_eq!( + message, + "field `model_source` expects `HuggingFace(repo, file)` or `LocalPath(path)`" + ); } #[test] @@ -156,8 +164,10 @@ mod tests { .expect_err("arity must fail") .to_string(); - assert!(message.contains("HuggingFace"), "got: {message}"); - assert!(message.contains("2 string"), "got: {message}"); + assert_eq!( + message, + "`HuggingFace` expects exactly 2 string arguments (repo, file); got 1" + ); } #[test] @@ -166,17 +176,22 @@ mod tests { .expect_err("arity must fail") .to_string(); - assert!(message.contains("LocalPath"), "got: {message}"); - assert!(message.contains("1 string"), "got: {message}"); + assert_eq!( + message, + "`LocalPath` expects exactly 1 string argument (path); got 2" + ); } #[test] - fn non_string_argument_is_rejected() { + fn huggingface_with_non_string_first_argument_is_rejected() { let message = parse("HuggingFace(42, \"file\")") .expect_err("non-string arg must fail") .to_string(); - assert!(message.contains("string literal"), "got: {message}"); + assert_eq!( + message, + "`model_source` argument `HuggingFace.repo` expects a string literal" + ); } #[test] @@ -185,7 +200,10 @@ mod tests { .expect_err("non-string second arg must fail") .to_string(); - assert!(message.contains("string literal"), "got: {message}"); + assert_eq!( + message, + "`model_source` argument `HuggingFace.file` expects a string literal" + ); } #[test] @@ -194,16 +212,9 @@ mod tests { .expect_err("non-string LocalPath arg must fail") .to_string(); - assert!(message.contains("string literal"), "got: {message}"); - } - - #[test] - fn unparseable_input_returns_err() { - let result = parse("@&^!"); - - assert!( - result.is_err(), - "garbage input must fail to parse as syn::Expr" + assert_eq!( + message, + "`model_source` argument `LocalPath.path` expects a string literal" ); } @@ -213,7 +224,10 @@ mod tests { .expect_err("non-path func must fail") .to_string(); - assert!(message.contains("HuggingFace"), "got: {message}"); + assert_eq!( + message, + "field `model_source` expects `HuggingFace(...)` or `LocalPath(...)`" + ); } #[test] @@ -222,7 +236,10 @@ mod tests { .expect_err("qualified path must fail") .to_string(); - assert!(message.contains("bare variant name"), "got: {message}"); + assert_eq!( + message, + "field `model_source` expects the bare variant name `HuggingFace` or `LocalPath`" + ); } #[test] diff --git a/llama-cpp-test-harness/Cargo.toml b/llama-cpp-test-harness/Cargo.toml index 5089cdc02..06c9b9c61 100644 --- a/llama-cpp-test-harness/Cargo.toml +++ b/llama-cpp-test-harness/Cargo.toml @@ -8,11 +8,11 @@ publish = false [dependencies] anyhow = { workspace = true } -hf-hub = { workspace = true } inventory = { workspace = true } libtest-mimic = { workspace = true } llama-cpp-bindings = { workspace = true } llama-cpp-test-harness-macros = { workspace = true } +llama-cpp-test-model-provisioning = { workspace = true } thiserror = { workspace = true } [features] @@ -23,7 +23,11 @@ vulkan = ["llama-cpp-bindings/vulkan"] rocm = ["llama-cpp-bindings/rocm"] [[test]] -name = "harness_self_test" +name = "duplicate_trial_names" +harness = false + +[[test]] +name = "phase_conclusions" harness = false [lints] diff --git a/llama-cpp-test-harness/src/context_params.rs b/llama-cpp-test-harness/src/context_params.rs index e135b9770..304f00b0e 100644 --- a/llama-cpp-test-harness/src/context_params.rs +++ b/llama-cpp-test-harness/src/context_params.rs @@ -59,52 +59,18 @@ mod tests { n_ctx: 1024, n_batch: 256, n_ubatch: 128, - ..BASELINE + n_seq_max: 4, + n_threads_batch: Some(8), + embeddings: true, } .into_llama_context_params(); assert_eq!(params.n_ctx(), NonZeroU32::new(1024)); assert_eq!(params.n_batch(), 256); assert_eq!(params.n_ubatch(), 128); - } - - #[test] - fn into_llama_context_params_propagates_embeddings_flag() { - let off = ContextParams { - embeddings: false, - ..BASELINE - } - .into_llama_context_params(); - let on = ContextParams { - embeddings: true, - ..BASELINE - } - .into_llama_context_params(); - - assert!(!off.embeddings()); - assert!(on.embeddings()); - } - - #[test] - fn into_llama_context_params_propagates_n_seq_max() { - let params = ContextParams { - n_seq_max: 4, - ..BASELINE - } - .into_llama_context_params(); - - assert_eq!(params.context_params.n_seq_max, 4); - } - - #[test] - fn into_llama_context_params_applies_n_threads_batch_when_some() { - let params = ContextParams { - n_threads_batch: Some(8), - ..BASELINE - } - .into_llama_context_params(); - - assert_eq!(params.context_params.n_threads_batch, 8); + assert_eq!(params.n_seq_max(), 4); + assert_eq!(params.n_threads_batch(), 8); + assert!(params.embeddings()); } #[test] diff --git a/llama-cpp-test-harness/src/download_model.rs b/llama-cpp-test-harness/src/download_model.rs deleted file mode 100644 index 7a194fa90..000000000 --- a/llama-cpp-test-harness/src/download_model.rs +++ /dev/null @@ -1,30 +0,0 @@ -use std::path::PathBuf; - -use anyhow::Result; - -/// # Errors -/// -/// Returns an error if the HF client cannot be built or the file cannot be downloaded -/// (e.g., the repo or file does not exist, or network access fails). -pub fn download_model(repo: &str, file: &str) -> Result { - let api = hf_hub::api::sync::ApiBuilder::from_env() - .with_progress(true) - .build()?; - Ok(api.model(repo.to_owned()).get(file)?) -} - -#[cfg(test)] -mod tests { - use hf_hub::api::sync::ApiError; - - use super::download_model; - - #[test] - fn missing_file_in_real_repo_returns_error() { - let download_failure = - download_model("unsloth/Qwen3.5-0.8B-GGUF", "this-file-does-not-exist.gguf") - .expect_err("a file missing from an existing repository must fail to download"); - - assert!(download_failure.is::()); - } -} diff --git a/llama-cpp-test-harness/src/execution_phase.rs b/llama-cpp-test-harness/src/execution_phase.rs index ed81fa6f6..e52f68c52 100644 --- a/llama-cpp-test-harness/src/execution_phase.rs +++ b/llama-cpp-test-harness/src/execution_phase.rs @@ -1,5 +1,6 @@ +use llama_cpp_test_model_provisioning::load_key::LoadKey; + use crate::llama_test_registration::LlamaTestRegistration; -use crate::load_key::LoadKey; pub struct ExecutionPhase { pub key: LoadKey, @@ -26,10 +27,10 @@ impl ExecutionPhase { #[cfg(test)] mod tests { - use crate::GgufSource; - use crate::LlamaLoadMode; - use crate::load_key::LoadKey; - use crate::model_load_params::ModelLoadParams; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use llama_cpp_test_model_provisioning::gguf_source::GgufSource; + use llama_cpp_test_model_provisioning::load_key::LoadKey; + use llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams; use super::ExecutionPhase; diff --git a/llama-cpp-test-harness/src/execution_plan.rs b/llama-cpp-test-harness/src/execution_plan.rs index e1c203a54..5315c3bdb 100644 --- a/llama-cpp-test-harness/src/execution_plan.rs +++ b/llama-cpp-test-harness/src/execution_plan.rs @@ -1,14 +1,33 @@ use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::sync::Arc; use libtest_mimic::Arguments; use libtest_mimic::Conclusion; + use llama_cpp_bindings::llama_backend::LlamaBackend; use crate::execution_phase::ExecutionPhase; +use crate::harness_run_error::HarnessRunError; use crate::llama_test_registration::LlamaTestRegistration; use crate::scheduled_phase::ScheduledPhase; +fn refuse_duplicate_trial_names( + registrations: &[&'static LlamaTestRegistration], +) -> Result<(), HarnessRunError> { + let mut trial_names = BTreeSet::new(); + + registrations.iter().try_for_each(|registration| { + if trial_names.insert(registration.name) { + Ok(()) + } else { + Err(HarnessRunError::DuplicateTrialName { + name: registration.name, + }) + } + }) +} + fn collect_inventory_registrations() -> Vec<&'static LlamaTestRegistration> { inventory::iter:: .into_iter() @@ -20,8 +39,15 @@ pub struct ExecutionPlan { } impl ExecutionPlan { - #[must_use] - pub fn from_registrations(registrations: &[&'static LlamaTestRegistration]) -> Self { + /// # Errors + /// + /// Returns [`HarnessRunError::DuplicateTrialName`] when two registrations share a trial name, + /// since their results could not be told apart. + pub fn from_registrations( + registrations: &[&'static LlamaTestRegistration], + ) -> Result { + refuse_duplicate_trial_names(registrations)?; + let mut by_key: BTreeMap<_, Vec<&'static LlamaTestRegistration>> = BTreeMap::new(); for registration in registrations { by_key @@ -34,20 +60,14 @@ impl ExecutionPlan { registrations.sort_by_key(|registration| registration.name); phases.push(ExecutionPhase { key, registrations }); } - Self { phases } + Ok(Self { phases }) } - #[must_use] - pub fn from_inventory() -> Self { - let registrations = collect_inventory_registrations(); - Self::from_registrations(®istrations) - } - - #[must_use] - pub fn requests_void_logs(&self) -> bool { - self.phases - .iter() - .any(|phase| phase.registrations.iter().any(|reg| reg.void_logs)) + /// # Errors + /// + /// Returns the [`HarnessRunError`] of [`Self::from_registrations`] for every registered trial. + pub fn from_inventory() -> Result { + Self::from_registrations(&collect_inventory_registrations()) } #[must_use] @@ -73,16 +93,18 @@ impl ExecutionPlan { #[cfg(test)] mod tests { - use crate::LlamaLoadMode; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use llama_cpp_test_model_provisioning::gguf_source::GgufSource; + use llama_cpp_test_model_provisioning::load_key::LoadKey; + use llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams; + + use super::ExecutionPlan; use crate::context_params::ContextParams; - use crate::gguf_source::GgufSource; + use crate::expected_outcome::ExpectedOutcome; + use crate::llama_test_fn::LlamaTestFn; use crate::llama_test_registration::LlamaTestRegistration; - use crate::load_key::LoadKey; - use crate::model_load_params::ModelLoadParams; use crate::no_op::no_op; - use super::ExecutionPlan; - const TRIVIAL_CONTEXT_PARAMS: ContextParams = ContextParams { n_ctx: 1, n_batch: 1, @@ -115,8 +137,8 @@ mod tests { }, }, context_params: TRIVIAL_CONTEXT_PARAMS, - void_logs: false, - func: no_op, + func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, }; static REG_BETA_B: LlamaTestRegistration = LlamaTestRegistration { name: "bravo", @@ -132,8 +154,8 @@ mod tests { }, }, context_params: TRIVIAL_CONTEXT_PARAMS, - void_logs: false, - func: no_op, + func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, }; static REG_ALPHA_Z: LlamaTestRegistration = LlamaTestRegistration { name: "zulu", @@ -149,8 +171,8 @@ mod tests { }, }, context_params: TRIVIAL_CONTEXT_PARAMS, - void_logs: false, - func: no_op, + func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, }; static REG_BETA_DIFFERENT_CONTEXT: LlamaTestRegistration = LlamaTestRegistration { name: "charlie", @@ -166,38 +188,36 @@ mod tests { }, }, context_params: ALTERNATE_CONTEXT_PARAMS, - void_logs: false, - func: no_op, + func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, }; - static REG_VOID_LOGS: LlamaTestRegistration = LlamaTestRegistration { - name: "void-logs-trial", - key: LoadKey { - model_source: GgufSource::HuggingFace { - repo: "beta", - file: "f", - }, - mmproj_source: None, - model_load_params: ModelLoadParams { - n_gpu_layers: 0, - load_mode: LlamaLoadMode::Mmap, - }, - }, - context_params: TRIVIAL_CONTEXT_PARAMS, - void_logs: true, - func: no_op, - }; + #[test] + fn registrations_sharing_a_trial_name_are_refused() { + assert_eq!( + ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_A]) + .err() + .map(|refusal| refusal.to_string()), + Some( + "two trials are named `alpha`; every trial needs its own name so its result \ + can be told apart" + .to_owned() + ) + ); + } #[test] fn from_registrations_with_empty_input_yields_empty_plan() { - let plan = ExecutionPlan::from_registrations(&[]); + let plan = ExecutionPlan::from_registrations(&[]) + .expect("registrations with distinct names form a plan"); assert!(plan.phases.is_empty()); } #[test] fn registrations_with_same_load_key_collapse_to_one_phase() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_B]); + let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_B]) + .expect("registrations with distinct names form a plan"); assert_eq!(plan.phases.len(), 1); assert_eq!(plan.phases[0].registrations.len(), 2); @@ -205,7 +225,8 @@ mod tests { #[test] fn registrations_with_distinct_load_keys_form_phases_in_load_key_sort_order() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_ALPHA_Z]); + let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_ALPHA_Z]) + .expect("registrations with distinct names form a plan"); assert_eq!(plan.phases.len(), 2); assert_eq!( @@ -226,30 +247,18 @@ mod tests { #[test] fn within_a_phase_registrations_sort_by_name() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_B, ®_BETA_A]); + let plan = ExecutionPlan::from_registrations(&[®_BETA_B, ®_BETA_A]) + .expect("registrations with distinct names form a plan"); assert_eq!(plan.phases.len(), 1); assert_eq!(plan.phases[0].registrations[0].name, "alpha"); assert_eq!(plan.phases[0].registrations[1].name, "bravo"); } - #[test] - fn requests_void_logs_false_when_no_registration_opts_in() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_ALPHA_Z]); - - assert!(!plan.requests_void_logs()); - } - - #[test] - fn requests_void_logs_true_when_any_registration_opts_in() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_VOID_LOGS]); - - assert!(plan.requests_void_logs()); - } - #[test] fn registrations_sharing_a_load_key_but_differing_context_params_stay_in_one_phase() { - let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_DIFFERENT_CONTEXT]); + let plan = ExecutionPlan::from_registrations(&[®_BETA_A, ®_BETA_DIFFERENT_CONTEXT]) + .expect("registrations with distinct names form a plan"); assert_eq!(plan.phases.len(), 1); assert_eq!(plan.phases[0].registrations.len(), 2); diff --git a/llama-cpp-test-harness/src/expected_failure.rs b/llama-cpp-test-harness/src/expected_failure.rs new file mode 100644 index 000000000..6859d6e11 --- /dev/null +++ b/llama-cpp-test-harness/src/expected_failure.rs @@ -0,0 +1,37 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +use llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind; + +use crate::trial_outcome::TrialOutcome; + +/// The failure a trial declares it exists to produce. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExpectedFailure { + PhaseSetup(PhaseSetupFailureKind), + Test, +} + +impl ExpectedFailure { + #[must_use] + pub fn matches(self, outcome: &TrialOutcome<'_>) -> bool { + match (self, outcome) { + (Self::PhaseSetup(expected_kind), TrialOutcome::PhaseSetupFailed(setup_failure)) => { + setup_failure.kind() == expected_kind + } + (Self::Test, TrialOutcome::TestFailed(_)) => true, + (Self::PhaseSetup(_), TrialOutcome::Succeeded | TrialOutcome::TestFailed(_)) + | (Self::Test, TrialOutcome::Succeeded | TrialOutcome::PhaseSetupFailed(_)) => false, + } + } +} + +impl Display for ExpectedFailure { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::PhaseSetup(kind) => write!(formatter, "{kind}"), + Self::Test => formatter.write_str("TestFailure"), + } + } +} diff --git a/llama-cpp-test-harness/src/expected_outcome.rs b/llama-cpp-test-harness/src/expected_outcome.rs new file mode 100644 index 000000000..36a5bedb8 --- /dev/null +++ b/llama-cpp-test-harness/src/expected_outcome.rs @@ -0,0 +1,184 @@ +use libtest_mimic::Failed; +use libtest_mimic::Trial; + +use crate::expected_failure::ExpectedFailure; +use crate::trial_outcome::TrialOutcome; + +/// How a trial is declared to end: by succeeding, or by producing one exact failure on purpose, +/// for the stated reason. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExpectedOutcome { + Success, + Failure { + failure: ExpectedFailure, + reason: &'static str, + }, +} + +impl ExpectedOutcome { + /// Marks a trial that is declared to fail with the failure it expects and why. + #[must_use] + pub fn label(self, trial: Trial) -> Trial { + match self { + Self::Success => trial, + Self::Failure { failure, reason } => { + trial.with_kind(format!("expected {failure}: {reason}")) + } + } + } + + /// # Errors + /// + /// Returns [`Failed`] when `outcome` is not the declared one: a failure of a trial declared to + /// succeed, or a success or a different failure of a trial declared to fail. + pub fn judge(self, outcome: TrialOutcome<'_>) -> Result<(), Failed> { + match self { + Self::Success => match outcome { + TrialOutcome::Succeeded => Ok(()), + failure @ (TrialOutcome::PhaseSetupFailed(_) | TrialOutcome::TestFailed(_)) => { + Err(Failed::from(failure)) + } + }, + Self::Failure { failure, .. } if failure.matches(&outcome) => Ok(()), + Self::Failure { failure, .. } => { + Err(Failed::from(format!("expected {failure}, but {outcome}"))) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use anyhow::anyhow; + + use llama_cpp_test_model_provisioning::phase_setup_error::PhaseSetupError; + use llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind; + + use super::ExpectedOutcome; + use crate::expected_failure::ExpectedFailure; + use crate::trial_outcome::TrialOutcome; + + const EXPECTED_SETUP_FAILURE: ExpectedOutcome = ExpectedOutcome::Failure { + failure: ExpectedFailure::PhaseSetup(PhaseSetupFailureKind::MmprojPathNotUnicode), + reason: "the mmproj path is not Unicode on purpose", + }; + + const EXPECTED_TEST_FAILURE: ExpectedOutcome = ExpectedOutcome::Failure { + failure: ExpectedFailure::Test, + reason: "the body bails on purpose", + }; + + fn setup_failure() -> PhaseSetupError { + PhaseSetupError::MmprojPathNotUnicode { + mmproj_path: PathBuf::from("/models/mmproj.gguf"), + } + } + + fn verdict(expected_outcome: ExpectedOutcome, outcome: TrialOutcome<'_>) -> Result<(), String> { + expected_outcome + .judge(outcome) + .map_err(|failed| failed.message().unwrap_or_default().to_owned()) + } + + #[test] + fn a_success_meets_an_expected_success() { + assert_eq!( + verdict(ExpectedOutcome::Success, TrialOutcome::Succeeded), + Ok(()) + ); + } + + #[test] + fn a_failure_misses_an_expected_success_with_what_failed() { + let setup_failure = setup_failure(); + + assert_eq!( + [ + verdict( + ExpectedOutcome::Success, + TrialOutcome::PhaseSetupFailed(&setup_failure) + ), + verdict( + ExpectedOutcome::Success, + TrialOutcome::TestFailed(anyhow!("the body failed")) + ), + ], + [ + Err( + "phase setup failed: the mmproj path /models/mmproj.gguf is not Unicode" + .to_owned() + ), + Err("the body failed".to_owned()), + ] + ); + } + + #[test] + fn the_declared_failure_meets_its_expectation() { + let setup_failure = setup_failure(); + + assert_eq!( + [ + verdict( + EXPECTED_SETUP_FAILURE, + TrialOutcome::PhaseSetupFailed(&setup_failure) + ), + verdict( + EXPECTED_TEST_FAILURE, + TrialOutcome::TestFailed(anyhow!("the body bailed")) + ), + ], + [Ok(()), Ok(())] + ); + } + + #[test] + fn a_success_misses_an_expected_failure() { + assert_eq!( + verdict(EXPECTED_SETUP_FAILURE, TrialOutcome::Succeeded), + Err("expected MmprojPathNotUnicode, but the trial succeeded".to_owned()) + ); + } + + #[test] + fn a_different_failure_misses_an_expected_failure() { + let setup_failure = setup_failure(); + + assert_eq!( + [ + verdict( + ExpectedOutcome::Failure { + failure: ExpectedFailure::PhaseSetup(PhaseSetupFailureKind::ModelDownload), + reason: "the repository does not exist on purpose", + }, + TrialOutcome::PhaseSetupFailed(&setup_failure) + ), + verdict( + EXPECTED_SETUP_FAILURE, + TrialOutcome::TestFailed(anyhow!("the body failed")) + ), + verdict( + EXPECTED_TEST_FAILURE, + TrialOutcome::PhaseSetupFailed(&setup_failure) + ), + verdict(EXPECTED_TEST_FAILURE, TrialOutcome::Succeeded), + ], + [ + Err( + "expected ModelDownload, but phase setup failed: the mmproj path \ + /models/mmproj.gguf is not Unicode" + .to_owned() + ), + Err("expected MmprojPathNotUnicode, but the body failed".to_owned()), + Err( + "expected TestFailure, but phase setup failed: the mmproj path \ + /models/mmproj.gguf is not Unicode" + .to_owned() + ), + Err("expected TestFailure, but the trial succeeded".to_owned()), + ] + ); + } +} diff --git a/llama-cpp-test-harness/src/gguf_source.rs b/llama-cpp-test-harness/src/gguf_source.rs deleted file mode 100644 index 71d12cd58..000000000 --- a/llama-cpp-test-harness/src/gguf_source.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::fmt; -use std::fmt::Display; -use std::fmt::Formatter; -use std::path::PathBuf; - -use anyhow::Context; -use anyhow::Result; - -use crate::download_model::download_model; - -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub enum GgufSource { - HuggingFace { - repo: &'static str, - file: &'static str, - }, - LocalPath(&'static str), -} - -impl Display for GgufSource { - fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { - match self { - Self::HuggingFace { repo, file } => write!(formatter, "{repo} / {file}"), - Self::LocalPath(path) => write!(formatter, "local:{path}"), - } - } -} - -impl GgufSource { - /// # Errors - /// - /// Returns an error if the HF download fails, with this source attached as its context. - /// `LocalPath` is infallible here — file existence is checked when the file is loaded. - pub fn resolve_path(self) -> Result { - match self { - Self::HuggingFace { repo, file } => download_model(repo, file).context(self), - Self::LocalPath(path) => Ok(PathBuf::from(path)), - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use super::GgufSource; - - #[test] - fn resolve_path_for_local_path_returns_the_literal_path() { - let source = GgufSource::LocalPath("/abs/example.gguf"); - - let resolved = source - .resolve_path() - .expect("LocalPath resolve is infallible"); - - assert_eq!(resolved, PathBuf::from("/abs/example.gguf")); - } - - #[test] - fn resolve_path_failure_names_the_huggingface_source() { - let source = GgufSource::HuggingFace { - repo: "intentee-test-harness/does-not-exist", - file: "no-such-file.gguf", - }; - - let download_failure = source - .resolve_path() - .expect_err("a file from a repository that does not exist must fail to download"); - - assert_eq!(download_failure.downcast_ref::(), Some(&source)); - } -} diff --git a/llama-cpp-test-harness/src/harness_arguments_error.rs b/llama-cpp-test-harness/src/harness_arguments_error.rs deleted file mode 100644 index f73d4ab7d..000000000 --- a/llama-cpp-test-harness/src/harness_arguments_error.rs +++ /dev/null @@ -1,9 +0,0 @@ -use thiserror::Error; - -#[derive(Debug, Eq, Error, PartialEq)] -pub enum HarnessArgumentsError { - #[error( - "the test harness requires --test-threads=1 (or unset); got --test-threads={requested}" - )] - ConflictingTestThreads { requested: usize }, -} diff --git a/llama-cpp-test-harness/src/harness_run_error.rs b/llama-cpp-test-harness/src/harness_run_error.rs index 55a21c5bd..dacae6587 100644 --- a/llama-cpp-test-harness/src/harness_run_error.rs +++ b/llama-cpp-test-harness/src/harness_run_error.rs @@ -1,12 +1,18 @@ -use llama_cpp_bindings::error::LlamaCppError; use thiserror::Error; -use crate::harness_arguments_error::HarnessArgumentsError; +use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; -#[derive(Debug, Error)] +#[derive(Debug, Eq, Error, PartialEq)] pub enum HarnessRunError { - #[error("failed to parse harness arguments: {0}")] - ArgumentParsing(#[from] HarnessArgumentsError), + #[error( + "the test harness requires --test-threads=1 (or unset); got --test-threads={requested}" + )] + ConflictingTestThreads { requested: usize }, + #[error( + "two trials are named `{name}`; every trial needs its own name so its result can be told \ + apart" + )] + DuplicateTrialName { name: &'static str }, #[error("failed to initialise the llama backend: {0}")] BackendInit(#[from] LlamaCppError), } diff --git a/llama-cpp-test-harness/src/lib.rs b/llama-cpp-test-harness/src/lib.rs index 7b59d0bbe..fbc107a0f 100644 --- a/llama-cpp-test-harness/src/lib.rs +++ b/llama-cpp-test-harness/src/lib.rs @@ -4,47 +4,18 @@ )] pub mod context_params; -pub mod download_model; pub mod execution_phase; pub mod execution_plan; -pub mod gguf_source; -pub mod harness_arguments_error; +pub mod expected_failure; +pub mod expected_outcome; pub mod harness_run_error; pub mod llama_fixture; pub mod llama_test_fn; pub mod llama_test_registration; pub mod llama_tests_main_macro; -pub mod load_key; -pub mod model_load_params; pub mod no_op; pub mod parse_harness_arguments; -pub mod phase_model_files; -pub mod phase_setup; -pub mod phase_state; pub mod run; pub mod run_to_conclusions; pub mod scheduled_phase; -#[cfg(test)] -mod test_backend_gate; - -pub use crate::context_params::ContextParams; -pub use crate::execution_phase::ExecutionPhase; -pub use crate::execution_plan::ExecutionPlan; -pub use crate::gguf_source::GgufSource; -pub use crate::llama_fixture::LlamaFixture; -pub use crate::llama_test_fn::LlamaTestFn; -pub use crate::llama_test_registration::LlamaTestRegistration; -pub use crate::load_key::LoadKey; -pub use crate::model_load_params::ModelLoadParams; -pub use crate::no_op::no_op; -pub use crate::phase_model_files::PhaseModelFiles; -pub use crate::phase_setup::PhaseSetup; -pub use crate::phase_state::PhaseState; -pub use crate::run::run; -pub use crate::run_to_conclusions::run_to_conclusions; -pub use crate::scheduled_phase::ScheduledPhase; -pub use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; -pub use llama_cpp_test_harness_macros::llama_test; - -#[doc(hidden)] -pub use inventory; +pub mod trial_outcome; diff --git a/llama-cpp-test-harness/src/llama_fixture.rs b/llama-cpp-test-harness/src/llama_fixture.rs index f427f8aa4..463ca61cc 100644 --- a/llama-cpp-test-harness/src/llama_fixture.rs +++ b/llama-cpp-test-harness/src/llama_fixture.rs @@ -1,10 +1,10 @@ use std::path::Path; -use anyhow::Result; use llama_cpp_bindings::context::LlamaContext; +use llama_cpp_bindings::error::llama_context_load_error::LlamaContextLoadError; use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; use crate::context_params::ContextParams; @@ -19,11 +19,11 @@ pub struct LlamaFixture<'fixture> { impl LlamaFixture<'_> { /// # Errors /// Forwards [`LlamaContext::from_model`] errors verbatim. - pub fn build_context(&self) -> Result> { - Ok(LlamaContext::from_model( + pub fn build_context(&self) -> Result, LlamaContextLoadError> { + LlamaContext::from_model( self.model, self.backend, (*self.context_params).into_llama_context_params(), - )?) + ) } } diff --git a/llama-cpp-test-harness/src/llama_test_fn.rs b/llama-cpp-test-harness/src/llama_test_fn.rs index f49249b6e..38521d21b 100644 --- a/llama-cpp-test-harness/src/llama_test_fn.rs +++ b/llama-cpp-test-harness/src/llama_test_fn.rs @@ -1,4 +1,28 @@ use crate::llama_fixture::LlamaFixture; -pub type LlamaTestFn = - for<'reference, 'fixture> fn(&'reference LlamaFixture<'fixture>) -> anyhow::Result<()>; +#[derive(Clone, Copy)] +pub enum LlamaTestFn { + Fallible( + for<'reference, 'fixture> fn(&'reference LlamaFixture<'fixture>) -> anyhow::Result<()>, + ), + Infallible(for<'reference, 'fixture> fn(&'reference LlamaFixture<'fixture>)), +} + +impl LlamaTestFn { + /// Runs the test against `fixture`; a test that cannot fail by returning an error succeeds + /// whenever it returns. + /// + /// # Errors + /// + /// Forwards the error a fallible test returns. + pub fn run(self, fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { + match self { + Self::Fallible(test) => test(fixture), + Self::Infallible(test) => { + test(fixture); + + Ok(()) + } + } + } +} diff --git a/llama-cpp-test-harness/src/llama_test_registration.rs b/llama-cpp-test-harness/src/llama_test_registration.rs index 2105120d5..5720e1a78 100644 --- a/llama-cpp-test-harness/src/llama_test_registration.rs +++ b/llama-cpp-test-harness/src/llama_test_registration.rs @@ -1,13 +1,17 @@ +use inventory::collect; + +use llama_cpp_test_model_provisioning::load_key::LoadKey; + use crate::context_params::ContextParams; +use crate::expected_outcome::ExpectedOutcome; use crate::llama_test_fn::LlamaTestFn; -use crate::load_key::LoadKey; pub struct LlamaTestRegistration { pub name: &'static str, pub key: LoadKey, pub context_params: ContextParams, - pub void_logs: bool, pub func: LlamaTestFn, + pub expected_outcome: ExpectedOutcome, } -inventory::collect!(LlamaTestRegistration); +collect!(LlamaTestRegistration); diff --git a/llama-cpp-test-harness/src/llama_tests_main_macro.rs b/llama-cpp-test-harness/src/llama_tests_main_macro.rs index 18ed6dab0..5ae5434a5 100644 --- a/llama-cpp-test-harness/src/llama_tests_main_macro.rs +++ b/llama-cpp-test-harness/src/llama_tests_main_macro.rs @@ -2,7 +2,7 @@ macro_rules! llama_tests_main { () => { fn main() -> ::std::process::ExitCode { - $crate::run() + $crate::run::run() } }; } diff --git a/llama-cpp-test-harness/src/no_op.rs b/llama-cpp-test-harness/src/no_op.rs index 95c62c541..1a746c447 100644 --- a/llama-cpp-test-harness/src/no_op.rs +++ b/llama-cpp-test-harness/src/no_op.rs @@ -1,8 +1,3 @@ use crate::llama_fixture::LlamaFixture; -/// # Errors -/// -/// Never; always returns `Ok(())`. The `Result` return type matches `LlamaTestFn`. -pub const fn no_op(_fixture: &LlamaFixture<'_>) -> anyhow::Result<()> { - Ok(()) -} +pub const fn no_op(_fixture: &LlamaFixture<'_>) {} diff --git a/llama-cpp-test-harness/src/parse_harness_arguments.rs b/llama-cpp-test-harness/src/parse_harness_arguments.rs index a1d6218da..1168ff879 100644 --- a/llama-cpp-test-harness/src/parse_harness_arguments.rs +++ b/llama-cpp-test-harness/src/parse_harness_arguments.rs @@ -1,33 +1,32 @@ use libtest_mimic::Arguments; -use crate::harness_arguments_error::HarnessArgumentsError; +use crate::harness_run_error::HarnessRunError; -fn validate(mut arguments: Arguments) -> Result { +fn validate(mut arguments: Arguments) -> Result { match arguments.test_threads { None | Some(1) => { arguments.test_threads = Some(1); Ok(arguments) } - Some(requested) => Err(HarnessArgumentsError::ConflictingTestThreads { requested }), + Some(requested) => Err(HarnessRunError::ConflictingTestThreads { requested }), } } /// # Errors /// -/// Returns [`HarnessArgumentsError::ConflictingTestThreads`] when `--test-threads` is set to +/// Returns [`HarnessRunError::ConflictingTestThreads`] when `--test-threads` is set to /// any value other than `1`. The harness orchestrates phase batching itself and cannot share /// that responsibility with `libtest_mimic`'s thread pool. -pub fn parse_harness_arguments() -> Result { - validate(Arguments::from_args()) +pub fn parse_harness_arguments(arguments: Arguments) -> Result { + validate(arguments) } #[cfg(test)] mod tests { use libtest_mimic::Arguments; - use crate::harness_arguments_error::HarnessArgumentsError; - use super::validate; + use crate::harness_run_error::HarnessRunError; #[test] fn validate_accepts_unset_test_threads_and_defaults_to_one() { @@ -58,7 +57,7 @@ mod tests { assert_eq!( error, - HarnessArgumentsError::ConflictingTestThreads { requested: 8 } + HarnessRunError::ConflictingTestThreads { requested: 8 } ); } diff --git a/llama-cpp-test-harness/src/phase_model_files.rs b/llama-cpp-test-harness/src/phase_model_files.rs deleted file mode 100644 index 6a467d47a..000000000 --- a/llama-cpp-test-harness/src/phase_model_files.rs +++ /dev/null @@ -1,101 +0,0 @@ -use std::path::PathBuf; -use std::sync::Arc; - -use anyhow::Result; -use llama_cpp_bindings::llama_backend::LlamaBackend; -use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::MtmdContext; -use llama_cpp_bindings::mtmd::MtmdContextParams; - -use crate::model_load_params::ModelLoadParams; -use crate::phase_state::PhaseState; - -#[derive(Debug)] -pub struct PhaseModelFiles { - pub model_path: PathBuf, - pub mmproj_path: Option, -} - -impl PhaseModelFiles { - /// # Errors - /// - /// Returns an error if loading the model into llama.cpp fails, or initializing the - /// `MtmdContext` from the mmproj file fails. - pub fn load_phase_state( - &self, - backend: &Arc, - model_load_params: ModelLoadParams, - ) -> Result { - let Self { - model_path, - mmproj_path, - } = self; - let model_params = model_load_params.into_llama_model_params(); - let model = LlamaModel::load_from_file(backend, model_path, &model_params)?; - - let mtmd_context = match mmproj_path { - Some(mmproj_path) => { - let mmproj_path_str = mmproj_path.to_string_lossy(); - let params = MtmdContextParams::default(); - Some(MtmdContext::init_from_file( - mmproj_path_str.as_ref(), - &model, - ¶ms, - )?) - } - None => None, - }; - - Ok(PhaseState { - mtmd_context, - model, - backend: Arc::clone(backend), - model_path: model_path.clone(), - }) - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - use std::sync::Arc; - use std::sync::PoisonError; - - use llama_cpp_bindings::LlamaModelLoadError; - use llama_cpp_bindings::llama_backend::LlamaBackend; - use llama_cpp_bindings::model::LlamaLoadMode; - - use crate::model_load_params::ModelLoadParams; - use crate::test_backend_gate::BACKEND_INIT_GATE; - - use super::PhaseModelFiles; - - const NON_GGUF_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml"); - - #[test] - fn load_phase_state_propagates_model_load_failure() { - let _gate = BACKEND_INIT_GATE - .lock() - .unwrap_or_else(PoisonError::into_inner); - let backend = Arc::new(LlamaBackend::init().expect("backend init must succeed")); - let model_files = PhaseModelFiles { - model_path: PathBuf::from(NON_GGUF_PATH), - mmproj_path: None, - }; - - let load_failure = model_files - .load_phase_state( - &backend, - ModelLoadParams { - n_gpu_layers: 0, - load_mode: LlamaLoadMode::Mmap, - }, - ) - .expect_err("a model file that is not GGUF must fail to load"); - - assert_eq!( - load_failure.downcast_ref::(), - Some(&LlamaModelLoadError::Unloadable) - ); - } -} diff --git a/llama-cpp-test-harness/src/run.rs b/llama-cpp-test-harness/src/run.rs index 69ffb165d..3d97e9998 100644 --- a/llama-cpp-test-harness/src/run.rs +++ b/llama-cpp-test-harness/src/run.rs @@ -1,5 +1,6 @@ use std::process::ExitCode; +use libtest_mimic::Arguments; use libtest_mimic::Conclusion; use crate::run_to_conclusions::run_to_conclusions; @@ -14,7 +15,7 @@ fn aggregate_exit_code(conclusions: &[Conclusion]) -> ExitCode { #[must_use] pub fn run() -> ExitCode { - match run_to_conclusions() { + match run_to_conclusions(Arguments::from_args()) { Ok(conclusions) => aggregate_exit_code(&conclusions), Err(error) => { eprintln!("llama-cpp-test-harness: {error}"); @@ -26,13 +27,12 @@ pub fn run() -> ExitCode { #[cfg(test)] mod tests { use std::process::ExitCode; + use std::sync::PoisonError; use libtest_mimic::Conclusion; - use llama_cpp_bindings::llama_backend::LlamaBackend; - use crate::harness_run_error::HarnessRunError; - use crate::run_to_conclusions::run_to_conclusions; - use crate::test_backend_gate::BACKEND_INIT_GATE; + use llama_cpp_bindings::llama_backend::LlamaBackend; + use llama_cpp_test_model_provisioning::backend_init_gate::BACKEND_INIT_GATE; use super::aggregate_exit_code; use super::run; @@ -57,58 +57,36 @@ mod tests { } } - fn as_u8(code: ExitCode) -> u8 { - let formatted = format!("{code:?}"); - formatted - .chars() - .filter(char::is_ascii_digit) - .collect::() - .parse::() - .unwrap_or(255) - } - #[test] fn aggregate_exit_code_zero_when_all_pass() { let code = aggregate_exit_code(&[passing_conclusion(), passing_conclusion()]); - assert_eq!(as_u8(code), 0); + assert_eq!(code, ExitCode::from(0)); } #[test] fn aggregate_exit_code_non_zero_when_any_fails() { let code = aggregate_exit_code(&[passing_conclusion(), failing_conclusion()]); - assert_eq!(as_u8(code), 101); + assert_eq!(code, ExitCode::from(101)); } #[test] - fn aggregate_exit_code_empty_input_succeeds() { + fn aggregate_exit_code_reports_success_when_no_phase_ran() { let code = aggregate_exit_code(&[]); - assert_eq!(as_u8(code), 0); - } - - #[test] - fn run_to_conclusions_errors_when_backend_init_fails() { - let _gate = BACKEND_INIT_GATE - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let _hold = LlamaBackend::init().expect("first init must succeed"); - - let outcome = run_to_conclusions(); - - assert!(matches!(outcome, Err(HarnessRunError::BackendInit(_)))); + assert_eq!(code, ExitCode::from(0)); } #[test] fn run_returns_exit_code_two_when_backend_init_fails() { let _gate = BACKEND_INIT_GATE .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); let _hold = LlamaBackend::init().expect("first init must succeed"); let code = run(); - assert_eq!(as_u8(code), 2); + assert_eq!(code, ExitCode::from(2)); } } diff --git a/llama-cpp-test-harness/src/run_to_conclusions.rs b/llama-cpp-test-harness/src/run_to_conclusions.rs index 0f422b4e3..6191bf24b 100644 --- a/llama-cpp-test-harness/src/run_to_conclusions.rs +++ b/llama-cpp-test-harness/src/run_to_conclusions.rs @@ -1,6 +1,8 @@ use std::sync::Arc; +use libtest_mimic::Arguments; use libtest_mimic::Conclusion; + use llama_cpp_bindings::llama_backend::LlamaBackend; use crate::execution_plan::ExecutionPlan; @@ -10,38 +12,62 @@ use crate::parse_harness_arguments::parse_harness_arguments; /// # Errors /// /// Returns [`HarnessRunError`] when the CLI arguments conflict with the harness's single-thread -/// requirement or the llama backend cannot be initialised. Surfacing these as a typed error keeps +/// requirement, two registered trials share a name, or the llama backend cannot be initialised. Surfacing these as a typed error keeps /// the failure explicit instead of aborting the process with a panic. -pub fn run_to_conclusions() -> Result, HarnessRunError> { - let arguments = parse_harness_arguments()?; - let mut backend = LlamaBackend::init()?; - let plan = ExecutionPlan::from_inventory(); - if plan.requests_void_logs() { - backend.void_logs(); - } - let backend = Arc::new(backend); +pub fn run_to_conclusions(arguments: Arguments) -> Result, HarnessRunError> { + let arguments = parse_harness_arguments(arguments)?; + let plan = ExecutionPlan::from_inventory()?; + let backend = Arc::new(LlamaBackend::init()?); Ok(plan.run(&backend, &arguments)) } #[cfg(test)] mod tests { - use crate::test_backend_gate::BACKEND_INIT_GATE; + use std::sync::PoisonError; + + use libtest_mimic::Arguments; + + use llama_cpp_bindings::error::llama_cpp_error::LlamaCppError; + use llama_cpp_bindings::llama_backend::LlamaBackend; + use llama_cpp_test_model_provisioning::backend_init_gate::BACKEND_INIT_GATE; use super::run_to_conclusions; + use crate::harness_run_error::HarnessRunError; + + #[test] + fn conflicting_arguments_stop_the_run_before_the_backend_starts() { + assert_eq!( + run_to_conclusions(Arguments { + test_threads: Some(8), + ..Arguments::default() + }) + .err(), + Some(HarnessRunError::ConflictingTestThreads { requested: 8 }) + ); + } #[test] - fn empty_inventory_yields_no_conclusions_and_skips_void_logs() { + fn an_already_initialized_backend_stops_the_run_with_a_backend_init_error() { let _gate = BACKEND_INIT_GATE .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(PoisonError::into_inner); + let _hold = LlamaBackend::init().expect("first init must succeed"); - let conclusions = - run_to_conclusions().expect("empty inventory must run without a setup failure"); - - assert!( - conclusions.is_empty(), - "empty inventory must yield zero phase conclusions" + assert_eq!( + run_to_conclusions(Arguments::default()).err(), + Some(HarnessRunError::BackendInit( + LlamaCppError::BackendAlreadyInitialized + )) ); } + + #[test] + fn empty_inventory_yields_no_conclusions() { + let _gate = BACKEND_INIT_GATE + .lock() + .unwrap_or_else(PoisonError::into_inner); + + assert_eq!(run_to_conclusions(Arguments::default()), Ok(Vec::new())); + } } diff --git a/llama-cpp-test-harness/src/scheduled_phase.rs b/llama-cpp-test-harness/src/scheduled_phase.rs index 5c4dfeef1..da71ea4f4 100644 --- a/llama-cpp-test-harness/src/scheduled_phase.rs +++ b/llama-cpp-test-harness/src/scheduled_phase.rs @@ -2,31 +2,42 @@ use std::sync::Arc; use libtest_mimic::Arguments; use libtest_mimic::Conclusion; -use libtest_mimic::Failed; use libtest_mimic::Trial; + use llama_cpp_bindings::llama_backend::LlamaBackend; +use llama_cpp_test_model_provisioning::phase_setup::PhaseSetup; use crate::execution_phase::ExecutionPhase; use crate::llama_fixture::LlamaFixture; use crate::llama_test_registration::LlamaTestRegistration; -use crate::phase_setup::PhaseSetup; - -fn trial(registration: &'static LlamaTestRegistration, setup: &Arc) -> Trial { - let setup = Arc::clone(setup); - - Trial::test(registration.name, move || match setup.load_phase_state() { - Ok(state) => (registration.func)(&LlamaFixture { +use crate::trial_outcome::TrialOutcome; + +fn trial_outcome<'setup>( + registration: &'static LlamaTestRegistration, + setup: &'setup PhaseSetup, +) -> TrialOutcome<'setup> { + match setup.load_phase_state() { + Ok(state) => match registration.func.run(&LlamaFixture { model: &state.model, backend: &state.backend, context_params: ®istration.context_params, mtmd_context: state.mtmd_context.as_ref(), model_path: &state.model_path, - }) - .map_err(|error| Failed::from(format!("{error:#}"))), - Err(setup_failure) => Err(Failed::from(format!( - "phase setup failed: {setup_failure:#}" - ))), - }) + }) { + Ok(()) => TrialOutcome::Succeeded, + Err(test_failure) => TrialOutcome::TestFailed(test_failure), + }, + Err(setup_failure) => TrialOutcome::PhaseSetupFailed(setup_failure), + } +} + +fn trial(registration: &'static LlamaTestRegistration, setup: &Arc) -> Trial { + let setup = Arc::clone(setup); + let expected_outcome = registration.expected_outcome; + + expected_outcome.label(Trial::test(registration.name, move || { + expected_outcome.judge(trial_outcome(registration, &setup)) + })) } pub struct ScheduledPhase<'plan> { @@ -89,55 +100,104 @@ mod tests { use std::sync::PoisonError; use libtest_mimic::Arguments; + use llama_cpp_bindings::llama_backend::LlamaBackend; - use llama_cpp_bindings::model::LlamaLoadMode; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use llama_cpp_test_model_provisioning::backend_init_gate::BACKEND_INIT_GATE; + use llama_cpp_test_model_provisioning::gguf_source::GgufSource; + use llama_cpp_test_model_provisioning::load_key::LoadKey; + use llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams; + use llama_cpp_test_model_provisioning::phase_setup_failure_kind::PhaseSetupFailureKind; + use super::ScheduledPhase; use crate::context_params::ContextParams; use crate::execution_phase::ExecutionPhase; - use crate::gguf_source::GgufSource; + use crate::expected_failure::ExpectedFailure; + use crate::expected_outcome::ExpectedOutcome; + use crate::llama_test_fn::LlamaTestFn; use crate::llama_test_registration::LlamaTestRegistration; - use crate::load_key::LoadKey; - use crate::model_load_params::ModelLoadParams; use crate::no_op::no_op; - use crate::test_backend_gate::BACKEND_INIT_GATE; - use super::ScheduledPhase; + const LOAD_KEY: LoadKey = LoadKey { + model_source: GgufSource::LocalPath("/nonexistent/llama-cpp-test-harness/model.gguf"), + mmproj_source: None, + model_load_params: ModelLoadParams { + n_gpu_layers: 0, + load_mode: LlamaLoadMode::Mmap, + }, + }; + + const CONTEXT_PARAMS: ContextParams = ContextParams { + n_ctx: 1, + n_batch: 1, + n_ubatch: 1, + n_seq_max: 1, + n_threads_batch: None, + embeddings: false, + }; static REGISTRATION: LlamaTestRegistration = LlamaTestRegistration { name: "scheduled_trial", - key: LoadKey { - model_source: GgufSource::LocalPath("/nonexistent/llama-cpp-test-harness/model.gguf"), - mmproj_source: None, - model_load_params: ModelLoadParams { - n_gpu_layers: 0, - load_mode: LlamaLoadMode::Mmap, - }, - }, - context_params: ContextParams { - n_ctx: 1, - n_batch: 1, - n_ubatch: 1, - n_seq_max: 1, - n_threads_batch: None, - embeddings: false, + key: LOAD_KEY, + context_params: CONTEXT_PARAMS, + func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, + }; + + static FAILING_ON_PURPOSE_REGISTRATION: LlamaTestRegistration = LlamaTestRegistration { + name: "failing_on_purpose", + key: LOAD_KEY, + context_params: CONTEXT_PARAMS, + func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Failure { + failure: ExpectedFailure::PhaseSetup(PhaseSetupFailureKind::ModelLoad), + reason: "the model file does not exist on purpose", }, - void_logs: false, - func: no_op, }; - fn selects_any_trial_with(harness_arguments: &[&str]) -> bool { + fn with_scheduled_phase) -> TResult>( + registrations: Vec<&'static LlamaTestRegistration>, + use_scheduled_phase: TUse, + ) -> TResult { let _gate = BACKEND_INIT_GATE .lock() .unwrap_or_else(PoisonError::into_inner); let backend = Arc::new(LlamaBackend::init().expect("backend init must succeed")); let phase = ExecutionPhase { - key: REGISTRATION.key, - registrations: vec![®ISTRATION], + key: LOAD_KEY, + registrations, }; + + use_scheduled_phase(&ScheduledPhase::new(&phase, &backend)) + } + + fn selects_any_trial_with(harness_arguments: &[&str]) -> bool { let arguments = Arguments::from_iter(iter::once("harness").chain(harness_arguments.iter().copied())); - ScheduledPhase::new(&phase, &backend).selects_any_trial(&arguments) + with_scheduled_phase(vec![®ISTRATION], |scheduled_phase| { + scheduled_phase.selects_any_trial(&arguments) + }) + } + + #[test] + fn only_a_trial_declared_to_fail_is_labelled_with_the_failure_it_expects_and_why() { + assert_eq!( + with_scheduled_phase( + vec![®ISTRATION, &FAILING_ON_PURPOSE_REGISTRATION], + |scheduled_phase| { + scheduled_phase + .trials + .iter() + .map(|trial| trial.kind().to_owned()) + .collect::>() + } + ), + [ + String::new(), + "expected ModelLoad: the model file does not exist on purpose".to_owned() + ] + ); } #[test] diff --git a/llama-cpp-test-harness/src/test_backend_gate.rs b/llama-cpp-test-harness/src/test_backend_gate.rs deleted file mode 100644 index c762fece8..000000000 --- a/llama-cpp-test-harness/src/test_backend_gate.rs +++ /dev/null @@ -1,2 +0,0 @@ -#[cfg(test)] -pub static BACKEND_INIT_GATE: std::sync::Mutex<()> = std::sync::Mutex::new(()); diff --git a/llama-cpp-test-harness/src/trial_outcome.rs b/llama-cpp-test-harness/src/trial_outcome.rs new file mode 100644 index 000000000..48a07511f --- /dev/null +++ b/llama-cpp-test-harness/src/trial_outcome.rs @@ -0,0 +1,27 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +use anyhow::Error; + +use llama_cpp_test_model_provisioning::phase_setup_error::PhaseSetupError; + +/// How a trial actually ended. +#[derive(Debug)] +pub enum TrialOutcome<'setup> { + Succeeded, + PhaseSetupFailed(&'setup PhaseSetupError), + TestFailed(Error), +} + +impl Display for TrialOutcome<'_> { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Succeeded => formatter.write_str("the trial succeeded"), + Self::PhaseSetupFailed(setup_failure) => { + write!(formatter, "phase setup failed: {setup_failure}") + } + Self::TestFailed(test_failure) => write!(formatter, "{test_failure:#}"), + } + } +} diff --git a/llama-cpp-test-harness/tests/duplicate_trial_names.rs b/llama-cpp-test-harness/tests/duplicate_trial_names.rs new file mode 100644 index 000000000..1bfed55f3 --- /dev/null +++ b/llama-cpp-test-harness/tests/duplicate_trial_names.rs @@ -0,0 +1,62 @@ +use std::process::ExitCode; + +use inventory::submit; +use libtest_mimic::Arguments; + +use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; +use llama_cpp_test_harness::context_params::ContextParams; +use llama_cpp_test_harness::expected_outcome::ExpectedOutcome; +use llama_cpp_test_harness::harness_run_error::HarnessRunError; +use llama_cpp_test_harness::llama_test_fn::LlamaTestFn; +use llama_cpp_test_harness::llama_test_registration::LlamaTestRegistration; +use llama_cpp_test_harness::no_op::no_op; +use llama_cpp_test_harness::run_to_conclusions::run_to_conclusions; +use llama_cpp_test_model_provisioning::gguf_source::GgufSource; +use llama_cpp_test_model_provisioning::load_key::LoadKey; +use llama_cpp_test_model_provisioning::model_load_params::ModelLoadParams; + +const NAME_REGISTERED_TWICE: &str = "a_trial_registered_twice_under_one_name"; + +const REGISTRATION: LlamaTestRegistration = LlamaTestRegistration { + name: NAME_REGISTERED_TWICE, + key: LoadKey { + model_source: GgufSource::LocalPath("Cargo.toml"), + mmproj_source: None, + model_load_params: ModelLoadParams { + n_gpu_layers: 0, + load_mode: LlamaLoadMode::Mmap, + }, + }, + context_params: ContextParams { + n_ctx: 1, + n_batch: 1, + n_ubatch: 1, + n_seq_max: 1, + n_threads_batch: None, + embeddings: false, + }, + func: LlamaTestFn::Infallible(no_op), + expected_outcome: ExpectedOutcome::Success, +}; + +submit! { REGISTRATION } + +submit! { REGISTRATION } + +fn main() -> ExitCode { + let refusal = run_to_conclusions(Arguments::from_args()).err(); + + if refusal + == Some(HarnessRunError::DuplicateTrialName { + name: NAME_REGISTERED_TWICE, + }) + { + ExitCode::SUCCESS + } else { + eprintln!( + "duplicate_trial_names: the harness did not refuse the duplicate name: {refusal:?}" + ); + + ExitCode::FAILURE + } +} diff --git a/llama-cpp-test-harness/tests/harness_self_test.rs b/llama-cpp-test-harness/tests/phase_conclusions.rs similarity index 57% rename from llama-cpp-test-harness/tests/harness_self_test.rs rename to llama-cpp-test-harness/tests/phase_conclusions.rs index 42e4f7b8c..b2b090246 100644 --- a/llama-cpp-test-harness/tests/harness_self_test.rs +++ b/llama-cpp-test-harness/tests/phase_conclusions.rs @@ -1,51 +1,46 @@ -#![expect( - clippy::unnecessary_wraps, - reason = "harness trial functions must return Result to satisfy LlamaTestFn registration" -)] use std::process::ExitCode; use anyhow::Result; use anyhow::bail; -use llama_cpp_test_harness::LlamaFixture; -use llama_cpp_test_harness::llama_test; -use llama_cpp_test_harness::no_op; -use llama_cpp_test_harness::run_to_conclusions; +use libtest_mimic::Arguments; + +use llama_cpp_test_harness::llama_fixture::LlamaFixture; +use llama_cpp_test_harness::no_op::no_op; +use llama_cpp_test_harness::run_to_conclusions::run_to_conclusions; +use llama_cpp_test_harness_macros::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, - void_logs = true, )] -fn phase_a_first_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { - let formatted = format!("{:?}", fixture.model); - assert!(formatted.contains("LlamaModel")); - no_op(fixture) +const fn phase_a_passing_trial_that_only_needs_its_model_to_load(fixture: &LlamaFixture<'_>) { + no_op(fixture); } #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64 )] -fn phase_a_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { +fn phase_a_second_passing_trial(fixture: &LlamaFixture<'_>) { assert_eq!(fixture.context_params.n_ctx, 512); - Ok(()) } #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, - n_ubatch = 64 + n_ubatch = 64, + expected_failure = TestFailure("the body bails to exercise reporting a failing test"), )] fn phase_a_intentionally_failing_trial(_fixture: &LlamaFixture<'_>) -> Result<()> { bail!("intentional failure to exercise the trial-failure dispatch path"); @@ -53,33 +48,31 @@ fn phase_a_intentionally_failing_trial(_fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64 )] -fn phase_b_first_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { +fn phase_b_first_passing_trial(fixture: &LlamaFixture<'_>) { assert!(fixture.mtmd_context.is_none()); - Ok(()) } #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64 )] -fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { - let _markers = fixture.model.tool_call_markers(); - Ok(()) +fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) { + assert_eq!(fixture.context_params.n_ubatch, 64); } #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, @@ -91,25 +84,37 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { load_mode = Mmap, n_ctx = 1, n_batch = 1, - n_ubatch = 1 + n_ubatch = 1, + expected_failure = SetupFailure( + ModelDownload, + "the model repository does not exist, so the phase cannot download its model" + ), )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, mmproj_source = HuggingFace("intentee-test-harness/does-not-exist", "no-such-mmproj.gguf"), + expected_failure = SetupFailure( + MmprojDownload, + "the mmproj repository does not exist, so the phase cannot download its mmproj" + ), )] #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, - mmproj_source = LocalPath("/nonexistent/llama-cpp-test-harness/no-such-mmproj.gguf"), + mmproj_source = LocalPath("/nonexistent/llama-cpp-test-harness/no-such-local-mmproj.gguf"), + expected_failure = SetupFailure( + MmprojLoad, + "the local mmproj file does not exist, so llama.cpp cannot load it" + ), )] #[llama_test( model_source = LocalPath("Cargo.toml"), @@ -117,67 +122,57 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { load_mode = Mmap, n_ctx = 1, n_batch = 1, - n_ubatch = 1 + n_ubatch = 1, + expected_failure = SetupFailure( + ModelLoad, + "Cargo.toml is not a GGUF model, so llama.cpp cannot load it" + ), )] -fn shared_setup_failure_and_phase_a_trial(fixture: &LlamaFixture<'_>) -> Result<()> { +fn phase_a_trial_also_scheduled_into_every_setup_failure_phase(fixture: &LlamaFixture<'_>) { assert!(fixture.model_path.exists()); - Ok(()) } #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), - n_gpu_layers = 999, + n_gpu_layers = -1, load_mode = Mmap, n_ctx = 512, n_batch = 128, n_ubatch = 64, mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf") )] -fn phase_d_mmproj_trial(fixture: &LlamaFixture<'_>) -> Result<()> { +fn phase_d_passing_trial_with_a_loaded_mmproj(fixture: &LlamaFixture<'_>) { assert!( fixture.mtmd_context.is_some(), "mmproj_file declared, but fixture.mtmd_context is None", ); - Ok(()) } const EXPECTED_PHASES: usize = 7; -const EXPECTED_PASSED: u64 = 6; -const EXPECTED_FAILED: u64 = 5; fn main() -> ExitCode { - let conclusions = match run_to_conclusions() { + let conclusions = match run_to_conclusions(Arguments::from_args()) { Ok(conclusions) => conclusions, Err(error) => { - eprintln!("harness_self_test: unexpected harness setup failure: {error}"); + eprintln!("phase_conclusions: unexpected harness setup failure: {error}"); return ExitCode::FAILURE; } }; let phases = conclusions.len(); - let total_passed: u64 = conclusions + let phases_with_a_failed_trial = conclusions .iter() - .map(|conclusion| conclusion.num_passed) - .sum(); - let total_failed: u64 = conclusions - .iter() - .map(|conclusion| conclusion.num_failed) - .sum(); + .filter(|conclusion| conclusion.has_failed()) + .count(); - if phases == EXPECTED_PHASES - && total_passed == EXPECTED_PASSED - && total_failed == EXPECTED_FAILED - { - eprintln!( - "harness_self_test: as expected — phases={phases}, passed={total_passed}, failed={total_failed}" - ); + if phases == EXPECTED_PHASES && phases_with_a_failed_trial == 0 { ExitCode::SUCCESS } else { eprintln!( - "harness_self_test: UNEXPECTED — phases={phases} (want {EXPECTED_PHASES}), \ - passed={total_passed} (want {EXPECTED_PASSED}), \ - failed={total_failed} (want {EXPECTED_FAILED})" + "phase_conclusions: want {EXPECTED_PHASES} phases without a failed trial; got \ + {phases} phases, {phases_with_a_failed_trial} of them with a failed trial" ); + ExitCode::FAILURE } } diff --git a/llama-cpp-test-model-provisioning/Cargo.toml b/llama-cpp-test-model-provisioning/Cargo.toml new file mode 100644 index 000000000..074cd44c6 --- /dev/null +++ b/llama-cpp-test-model-provisioning/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "llama-cpp-test-model-provisioning" +description = "Downloads and loads the models llama-cpp-bindings integration tests run against" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[dependencies] +hf-hub = { workspace = true } +llama-cpp-bindings = { workspace = true } +thiserror = { workspace = true } + +[features] +cuda = ["llama-cpp-bindings/cuda"] +cuda-no-vmm = ["llama-cpp-bindings/cuda-no-vmm"] +metal = ["llama-cpp-bindings/metal"] +vulkan = ["llama-cpp-bindings/vulkan"] +rocm = ["llama-cpp-bindings/rocm"] + +[lints] +workspace = true diff --git a/llama-cpp-test-model-provisioning/src/backend_init_gate.rs b/llama-cpp-test-model-provisioning/src/backend_init_gate.rs new file mode 100644 index 000000000..61f6765c4 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/backend_init_gate.rs @@ -0,0 +1,5 @@ +use std::sync::Mutex; + +/// Serializes the tests of one process that initialize the llama backend, which llama.cpp allows +/// only once at a time. +pub static BACKEND_INIT_GATE: Mutex<()> = Mutex::new(()); diff --git a/llama-cpp-test-model-provisioning/src/download_model.rs b/llama-cpp-test-model-provisioning/src/download_model.rs new file mode 100644 index 000000000..c88d03833 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/download_model.rs @@ -0,0 +1,15 @@ +use std::path::PathBuf; + +use hf_hub::api::sync::ApiBuilder; +use hf_hub::api::sync::ApiError; + +/// # Errors +/// +/// Returns the [`ApiError`] of building the HF client or of downloading the file, e.g. when the +/// repository or file does not exist, or network access fails. +pub fn download_model(repo: &str, file: &str) -> Result { + ApiBuilder::from_env() + .with_progress(true) + .build() + .and_then(|api| api.model(repo.to_owned()).get(file)) +} diff --git a/llama-cpp-test-model-provisioning/src/gguf_source.rs b/llama-cpp-test-model-provisioning/src/gguf_source.rs new file mode 100644 index 000000000..3cb2be25a --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/gguf_source.rs @@ -0,0 +1,39 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; +use std::path::PathBuf; + +use hf_hub::api::sync::ApiError; + +use crate::download_model::download_model; + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum GgufSource { + HuggingFace { + repo: &'static str, + file: &'static str, + }, + LocalPath(&'static str), +} + +impl Display for GgufSource { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::HuggingFace { repo, file } => write!(formatter, "{repo} / {file}"), + Self::LocalPath(path) => write!(formatter, "local:{path}"), + } + } +} + +impl GgufSource { + /// # Errors + /// + /// Returns the [`ApiError`] of downloading a `HuggingFace` source. `LocalPath` is infallible + /// here; whether the file exists is checked when it is loaded. + pub fn resolve_path(self) -> Result { + match self { + Self::HuggingFace { repo, file } => download_model(repo, file), + Self::LocalPath(path) => Ok(PathBuf::from(path)), + } + } +} diff --git a/llama-cpp-test-model-provisioning/src/lib.rs b/llama-cpp-test-model-provisioning/src/lib.rs new file mode 100644 index 000000000..9ecbfeca9 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/lib.rs @@ -0,0 +1,15 @@ +#![cfg_attr( + not(test), + deny(clippy::unwrap_used, clippy::expect_used, clippy::panic) +)] + +pub mod backend_init_gate; +pub mod download_model; +pub mod gguf_source; +pub mod load_key; +pub mod model_load_params; +pub mod phase_model_files; +pub mod phase_setup; +pub mod phase_setup_error; +pub mod phase_setup_failure_kind; +pub mod phase_state; diff --git a/llama-cpp-test-harness/src/load_key.rs b/llama-cpp-test-model-provisioning/src/load_key.rs similarity index 61% rename from llama-cpp-test-harness/src/load_key.rs rename to llama-cpp-test-model-provisioning/src/load_key.rs index 2f889f49e..964f070f9 100644 --- a/llama-cpp-test-harness/src/load_key.rs +++ b/llama-cpp-test-model-provisioning/src/load_key.rs @@ -1,8 +1,7 @@ -use anyhow::Result; - use crate::gguf_source::GgufSource; use crate::model_load_params::ModelLoadParams; use crate::phase_model_files::PhaseModelFiles; +use crate::phase_setup_error::PhaseSetupError; #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct LoadKey { @@ -14,13 +13,31 @@ pub struct LoadKey { impl LoadKey { /// # Errors /// - /// Returns an error if downloading the model file or the mmproj file fails. - pub fn download_model_files(&self) -> Result { + /// Returns [`PhaseSetupError::ModelDownload`] or [`PhaseSetupError::MmprojDownload`] when the + /// model or the mmproj file cannot be downloaded. + pub fn download_model_files(&self) -> Result { + let Self { + model_source, + mmproj_source, + .. + } = *self; + Ok(PhaseModelFiles { - model_path: self.model_source.resolve_path()?, - mmproj_path: self - .mmproj_source - .map(GgufSource::resolve_path) + model_path: model_source.resolve_path().map_err(|cause| { + PhaseSetupError::ModelDownload { + model_source, + cause, + } + })?, + mmproj_path: mmproj_source + .map(|mmproj_source| { + mmproj_source + .resolve_path() + .map_err(|cause| PhaseSetupError::MmprojDownload { + mmproj_source, + cause, + }) + }) .transpose()?, }) } @@ -30,12 +47,12 @@ impl LoadKey { mod tests { use std::path::PathBuf; - use llama_cpp_bindings::model::LlamaLoadMode; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use super::LoadKey; use crate::gguf_source::GgufSource; use crate::model_load_params::ModelLoadParams; - - use super::LoadKey; + use crate::phase_setup_failure_kind::PhaseSetupFailureKind; fn baseline() -> LoadKey { LoadKey { @@ -89,7 +106,7 @@ mod tests { #[test] fn different_model_load_params_compare_unequal() { let mut other = baseline(); - other.model_load_params.n_gpu_layers = 999; + other.model_load_params.n_gpu_layers = -1; assert_ne!(baseline(), other); } @@ -112,22 +129,35 @@ mod tests { } #[test] - fn download_model_files_propagates_mmproj_download_failure() { - let missing_mmproj_source = GgufSource::HuggingFace { + fn a_model_that_cannot_be_downloaded_fails_as_a_model_download() { + let mut key = baseline(); + key.model_source = GgufSource::HuggingFace { repo: "intentee-test-harness/does-not-exist", - file: "no-such-mmproj.gguf", + file: "no-such-file.gguf", }; + + assert_eq!( + key.download_model_files() + .err() + .map(|setup_failure| setup_failure.kind()), + Some(PhaseSetupFailureKind::ModelDownload) + ); + } + + #[test] + fn an_mmproj_that_cannot_be_downloaded_fails_as_an_mmproj_download() { let mut key = baseline(); key.model_source = GgufSource::LocalPath("/abs/model.gguf"); - key.mmproj_source = Some(missing_mmproj_source); - - let download_failure = key - .download_model_files() - .expect_err("an mmproj from a repository that does not exist must fail to download"); + key.mmproj_source = Some(GgufSource::HuggingFace { + repo: "intentee-test-harness/does-not-exist", + file: "no-such-mmproj.gguf", + }); assert_eq!( - download_failure.downcast_ref::(), - Some(&missing_mmproj_source) + key.download_model_files() + .err() + .map(|setup_failure| setup_failure.kind()), + Some(PhaseSetupFailureKind::MmprojDownload) ); } } diff --git a/llama-cpp-test-harness/src/model_load_params.rs b/llama-cpp-test-model-provisioning/src/model_load_params.rs similarity index 96% rename from llama-cpp-test-harness/src/model_load_params.rs rename to llama-cpp-test-model-provisioning/src/model_load_params.rs index 5ee9e2433..6e2be6ef0 100644 --- a/llama-cpp-test-harness/src/model_load_params.rs +++ b/llama-cpp-test-model-provisioning/src/model_load_params.rs @@ -27,7 +27,7 @@ mod tests { use super::ModelLoadParams; #[test] - fn into_llama_model_params_carries_all_three_fields() { + fn into_llama_model_params_carries_both_fields() { let params = ModelLoadParams { n_gpu_layers: 7, load_mode: LlamaLoadMode::Mlock, diff --git a/llama-cpp-test-model-provisioning/src/phase_model_files.rs b/llama-cpp-test-model-provisioning/src/phase_model_files.rs new file mode 100644 index 000000000..848157e06 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/phase_model_files.rs @@ -0,0 +1,136 @@ +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use llama_cpp_bindings::llama_backend::LlamaBackend; +use llama_cpp_bindings::model::LlamaModel; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_context_params::MtmdContextParams; + +use crate::model_load_params::ModelLoadParams; +use crate::phase_setup_error::PhaseSetupError; +use crate::phase_state::PhaseState; + +fn unicode_mmproj_path(mmproj_path: &Path) -> Result<&str, PhaseSetupError> { + mmproj_path + .to_str() + .ok_or_else(|| PhaseSetupError::MmprojPathNotUnicode { + mmproj_path: mmproj_path.to_path_buf(), + }) +} + +#[derive(Debug)] +pub struct PhaseModelFiles { + pub model_path: PathBuf, + pub mmproj_path: Option, +} + +impl PhaseModelFiles { + /// # Errors + /// + /// Returns [`PhaseSetupError::MmprojPathNotUnicode`] before loading anything when the mmproj + /// path cannot be handed to llama.cpp, [`PhaseSetupError::ModelLoad`] when llama.cpp cannot + /// load the model, or [`PhaseSetupError::MmprojLoad`] when it cannot load the mmproj. + pub fn load_phase_state( + &self, + backend: &Arc, + model_load_params: ModelLoadParams, + ) -> Result { + let Self { + model_path, + mmproj_path, + } = self; + let unicode_mmproj_path = mmproj_path + .as_deref() + .map(unicode_mmproj_path) + .transpose()?; + let model = LlamaModel::load_from_file( + backend, + model_path, + &model_load_params.into_llama_model_params(), + ) + .map_err(|cause| PhaseSetupError::ModelLoad { + model_path: model_path.clone(), + cause, + })?; + let mtmd_context = unicode_mmproj_path + .map(|unicode_mmproj_path| { + MtmdContext::init_from_file( + unicode_mmproj_path, + &model, + &MtmdContextParams::default(), + ) + .map_err(|cause| PhaseSetupError::MmprojLoad { + mmproj_path: PathBuf::from(unicode_mmproj_path), + cause, + }) + }) + .transpose()?; + + Ok(PhaseState { + mtmd_context, + model, + backend: Arc::clone(backend), + model_path: model_path.clone(), + }) + } +} + +#[cfg(test)] +mod tests { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + use std::path::PathBuf; + use std::sync::Arc; + use std::sync::PoisonError; + + use llama_cpp_bindings::llama_backend::LlamaBackend; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + + use super::PhaseModelFiles; + use crate::backend_init_gate::BACKEND_INIT_GATE; + use crate::model_load_params::ModelLoadParams; + use crate::phase_setup_failure_kind::PhaseSetupFailureKind; + + const NON_GGUF_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml"); + + fn load_failure_kind(model_files: &PhaseModelFiles) -> Option { + let _gate = BACKEND_INIT_GATE + .lock() + .unwrap_or_else(PoisonError::into_inner); + let backend = Arc::new(LlamaBackend::init().expect("backend init must succeed")); + + model_files + .load_phase_state( + &backend, + ModelLoadParams { + n_gpu_layers: 0, + load_mode: LlamaLoadMode::Mmap, + }, + ) + .err() + .map(|setup_failure| setup_failure.kind()) + } + + #[test] + fn a_model_file_that_is_not_gguf_fails_as_a_model_load() { + assert_eq!( + load_failure_kind(&PhaseModelFiles { + model_path: PathBuf::from(NON_GGUF_PATH), + mmproj_path: None, + }), + Some(PhaseSetupFailureKind::ModelLoad) + ); + } + + #[test] + fn a_non_unicode_mmproj_path_is_refused_before_the_model_loads() { + assert_eq!( + load_failure_kind(&PhaseModelFiles { + model_path: PathBuf::from(NON_GGUF_PATH), + mmproj_path: Some(PathBuf::from(OsString::from_vec(vec![0xff]))), + }), + Some(PhaseSetupFailureKind::MmprojPathNotUnicode) + ); + } +} diff --git a/llama-cpp-test-harness/src/phase_setup.rs b/llama-cpp-test-model-provisioning/src/phase_setup.rs similarity index 90% rename from llama-cpp-test-harness/src/phase_setup.rs rename to llama-cpp-test-model-provisioning/src/phase_setup.rs index 9b70eade2..ca29caea1 100644 --- a/llama-cpp-test-harness/src/phase_setup.rs +++ b/llama-cpp-test-model-provisioning/src/phase_setup.rs @@ -1,19 +1,18 @@ use std::sync::Arc; use std::sync::OnceLock; -use anyhow::Error; -use anyhow::Result; use llama_cpp_bindings::llama_backend::LlamaBackend; use crate::load_key::LoadKey; use crate::phase_model_files::PhaseModelFiles; +use crate::phase_setup_error::PhaseSetupError; use crate::phase_state::PhaseState; pub struct PhaseSetup { key: LoadKey, backend: Arc, - model_files: OnceLock>, - phase_state: OnceLock>, + model_files: OnceLock>, + phase_state: OnceLock>, } impl PhaseSetup { @@ -31,7 +30,7 @@ impl PhaseSetup { /// /// Returns the failure of downloading the model or the mmproj file. The outcome is cached, so /// the files are downloaded at most once per phase. - pub fn download_model_files(&self) -> Result<&PhaseModelFiles, &Error> { + pub fn download_model_files(&self) -> Result<&PhaseModelFiles, &PhaseSetupError> { self.model_files .get_or_init(|| self.key.download_model_files()) .as_ref() @@ -41,7 +40,7 @@ impl PhaseSetup { /// /// Returns the cached download failure, or the failure of loading the downloaded files into /// llama.cpp. The outcome is cached, so the model is loaded at most once per phase. - pub fn load_phase_state(&self) -> Result<&PhaseState, &Error> { + pub fn load_phase_state(&self) -> Result<&PhaseState, &PhaseSetupError> { let model_files = self.download_model_files()?; self.phase_state @@ -57,14 +56,13 @@ mod tests { use std::sync::PoisonError; use llama_cpp_bindings::llama_backend::LlamaBackend; - use llama_cpp_bindings::model::LlamaLoadMode; + use llama_cpp_bindings::model::llama_load_mode::LlamaLoadMode; + use super::PhaseSetup; + use crate::backend_init_gate::BACKEND_INIT_GATE; use crate::gguf_source::GgufSource; use crate::load_key::LoadKey; use crate::model_load_params::ModelLoadParams; - use crate::test_backend_gate::BACKEND_INIT_GATE; - - use super::PhaseSetup; const NON_GGUF_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml"); diff --git a/llama-cpp-test-model-provisioning/src/phase_setup_error.rs b/llama-cpp-test-model-provisioning/src/phase_setup_error.rs new file mode 100644 index 000000000..1705a78d1 --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/phase_setup_error.rs @@ -0,0 +1,51 @@ +use std::path::PathBuf; + +use hf_hub::api::sync::ApiError; +use llama_cpp_bindings::error::llama_model_load_error::LlamaModelLoadError; +use llama_cpp_bindings::mtmd::mtmd_init_error::MtmdInitError; + +use crate::gguf_source::GgufSource; +use crate::phase_setup_failure_kind::PhaseSetupFailureKind; + +#[derive(Debug, thiserror::Error)] +pub enum PhaseSetupError { + #[error("the model {model_source} could not be downloaded: {cause}")] + ModelDownload { + model_source: GgufSource, + #[source] + cause: ApiError, + }, + #[error("the mmproj {mmproj_source} could not be downloaded: {cause}")] + MmprojDownload { + mmproj_source: GgufSource, + #[source] + cause: ApiError, + }, + #[error("the mmproj path {} is not Unicode", mmproj_path.display())] + MmprojPathNotUnicode { mmproj_path: PathBuf }, + #[error("the model {} could not be loaded: {cause}", model_path.display())] + ModelLoad { + model_path: PathBuf, + #[source] + cause: LlamaModelLoadError, + }, + #[error("the mmproj {} could not be loaded: {cause}", mmproj_path.display())] + MmprojLoad { + mmproj_path: PathBuf, + #[source] + cause: MtmdInitError, + }, +} + +impl PhaseSetupError { + #[must_use] + pub const fn kind(&self) -> PhaseSetupFailureKind { + match self { + Self::ModelDownload { .. } => PhaseSetupFailureKind::ModelDownload, + Self::MmprojDownload { .. } => PhaseSetupFailureKind::MmprojDownload, + Self::MmprojPathNotUnicode { .. } => PhaseSetupFailureKind::MmprojPathNotUnicode, + Self::ModelLoad { .. } => PhaseSetupFailureKind::ModelLoad, + Self::MmprojLoad { .. } => PhaseSetupFailureKind::MmprojLoad, + } + } +} diff --git a/llama-cpp-test-model-provisioning/src/phase_setup_failure_kind.rs b/llama-cpp-test-model-provisioning/src/phase_setup_failure_kind.rs new file mode 100644 index 000000000..729fae7cc --- /dev/null +++ b/llama-cpp-test-model-provisioning/src/phase_setup_failure_kind.rs @@ -0,0 +1,27 @@ +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +/// Which step of setting up a phase failed, as a test declares the failure it expects. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PhaseSetupFailureKind { + ModelDownload, + MmprojDownload, + MmprojPathNotUnicode, + ModelLoad, + MmprojLoad, +} + +impl Display for PhaseSetupFailureKind { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + let name = match self { + Self::ModelDownload => "ModelDownload", + Self::MmprojDownload => "MmprojDownload", + Self::MmprojPathNotUnicode => "MmprojPathNotUnicode", + Self::ModelLoad => "ModelLoad", + Self::MmprojLoad => "MmprojLoad", + }; + + formatter.write_str(name) + } +} diff --git a/llama-cpp-test-harness/src/phase_state.rs b/llama-cpp-test-model-provisioning/src/phase_state.rs similarity index 84% rename from llama-cpp-test-harness/src/phase_state.rs rename to llama-cpp-test-model-provisioning/src/phase_state.rs index 53e4560f3..1e70a10d9 100644 --- a/llama-cpp-test-harness/src/phase_state.rs +++ b/llama-cpp-test-model-provisioning/src/phase_state.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use llama_cpp_bindings::llama_backend::LlamaBackend; use llama_cpp_bindings::model::LlamaModel; -use llama_cpp_bindings::mtmd::MtmdContext; +use llama_cpp_bindings::mtmd::mtmd_context::MtmdContext; #[derive(Debug)] pub struct PhaseState { diff --git a/llama-cpp-wrapper-error-fixture/src/lib.rs b/llama-cpp-wrapper-error-fixture/src/lib.rs index fb54cf139..6e5f36bec 100644 --- a/llama-cpp-wrapper-error-fixture/src/lib.rs +++ b/llama-cpp-wrapper-error-fixture/src/lib.rs @@ -1,3 +1 @@ pub mod wrapper_allocated_error_message_pointer; - -pub use wrapper_allocated_error_message_pointer::wrapper_allocated_error_message_pointer; diff --git a/llama-cpp-wrapper-error-fixture/src/wrapper_allocated_error_message_pointer.rs b/llama-cpp-wrapper-error-fixture/src/wrapper_allocated_error_message_pointer.rs index 60e414ae4..3342fa8e9 100644 --- a/llama-cpp-wrapper-error-fixture/src/wrapper_allocated_error_message_pointer.rs +++ b/llama-cpp-wrapper-error-fixture/src/wrapper_allocated_error_message_pointer.rs @@ -20,7 +20,6 @@ pub fn wrapper_allocated_error_message_pointer() -> *mut c_char { let status = unsafe { llama_cpp_bindings_sys::llama_rs_json_schema_to_grammar( schema.as_ptr(), - false, &raw mut out_grammar, &raw mut out_error, ) @@ -28,7 +27,7 @@ pub fn wrapper_allocated_error_message_pointer() -> *mut c_char { assert_eq!( status, - llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_LLAMA_CPP_THREW_CXX_EXCEPTION, + llama_cpp_bindings_sys::LLAMA_RS_JSON_SCHEMA_TO_GRAMMAR_INVALID_JSON, "the fixture depends on the llama.cpp json parser rejecting this schema" ); assert!( diff --git a/llama-cpp-wrapper-sources/src/compile_commands_file.rs b/llama-cpp-wrapper-sources/src/compile_commands_file.rs index 2744876f5..6e0cc8c44 100644 --- a/llama-cpp-wrapper-sources/src/compile_commands_file.rs +++ b/llama-cpp-wrapper-sources/src/compile_commands_file.rs @@ -4,6 +4,8 @@ use std::path::PathBuf; use crate::compile_command::CompileCommand; use crate::cpp_standard::CPP_STANDARD; +use crate::fault_injection_define::FAULT_INJECTION_DEFINE; +use crate::system_include_flags::system_include_flags; use crate::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use crate::wrapper_source_paths::wrapper_source_paths; use crate::wrapper_sources_error::WrapperSourcesError; @@ -25,12 +27,18 @@ impl CompileCommandsFile { wrapper_source_paths(&self.sys_dir) .into_iter() .map(|file| { - let mut arguments = vec![CPP_COMPILER.to_owned(), format!("-std={CPP_STANDARD}")]; + let mut arguments = vec![ + CPP_COMPILER.to_owned(), + format!("-std={CPP_STANDARD}"), + format!("-D{FAULT_INJECTION_DEFINE}"), + ]; for include_dir in WRAPPER_INCLUDE_DIRS { arguments.push(format!("-I{}", absolute_path(&self.sys_dir, include_dir))); } + arguments.extend(system_include_flags(&self.sys_dir)); + arguments.push("-c".to_owned()); arguments.push(file.display().to_string()); @@ -69,9 +77,9 @@ mod tests { use std::path::PathBuf; use super::CompileCommandsFile; + use crate::system_include_flags::system_include_flags; use crate::wrapper_include_dirs::WRAPPER_INCLUDE_DIRS; use crate::wrapper_source_paths::wrapper_source_paths; - use crate::wrapper_sources_error::WrapperSourcesError; fn database_for(output_path: &str) -> CompileCommandsFile { CompileCommandsFile { @@ -122,7 +130,28 @@ mod tests { } #[test] - fn the_compiler_and_standard_lead_the_command_line() { + fn every_command_includes_the_vendored_headers_as_system_headers() { + let commands = database_for("/unused").commands(); + + assert_eq!( + commands + .iter() + .map(|command| command + .arguments + .iter() + .filter(|argument| argument.starts_with("-isystem")) + .cloned() + .collect::>()) + .collect::>>(), + vec![ + system_include_flags(Path::new("/repo/llama-cpp-bindings-sys")); + wrapper_source_paths(Path::new("/repo/llama-cpp-bindings-sys")).len() + ] + ); + } + + #[test] + fn the_compiler_standard_and_fault_injection_lead_the_command_line() { let commands = database_for("/unused").commands(); assert_eq!( @@ -131,30 +160,46 @@ mod tests { .map(|command| command .arguments .iter() - .take(2) + .take(3) .cloned() .collect::>()) .unwrap_or_default(), - vec!["c++".to_owned(), "-std=c++17".to_owned()] + vec![ + "c++".to_owned(), + "-std=c++17".to_owned(), + "-DLLAMA_RS_FAULT_INJECTION".to_owned() + ] ); } #[test] fn creating_the_database_under_a_missing_directory_reports_the_path() { - let output_path = "/nonexistent-directory/compile_commands.json"; - - assert!(matches!( - database_for(output_path).write(), - Err(WrapperSourcesError::Create { ref path, .. }) if path == Path::new(output_path) - )); + assert_eq!( + database_for("/nonexistent-directory/compile_commands.json") + .write() + .err() + .map(|error| error.to_string()), + Some( + "compilation database file /nonexistent-directory/compile_commands.json could \ + not be created: No such file or directory (os error 2)" + .to_owned() + ) + ); } #[cfg(target_os = "linux")] #[test] fn a_destination_that_cannot_absorb_the_bytes_reports_the_path() { - assert!(matches!( - database_for("/dev/full").write(), - Err(WrapperSourcesError::Serialize { ref path, .. }) if path == Path::new("/dev/full") - )); + assert_eq!( + database_for("/dev/full") + .write() + .err() + .map(|error| error.to_string()), + Some( + "compilation database could not be written to /dev/full: No space left on \ + device (os error 28)" + .to_owned() + ) + ); } } diff --git a/llama-cpp-wrapper-sources/src/fault_injection_define.rs b/llama-cpp-wrapper-sources/src/fault_injection_define.rs new file mode 100644 index 000000000..56cb4db5c --- /dev/null +++ b/llama-cpp-wrapper-sources/src/fault_injection_define.rs @@ -0,0 +1 @@ +pub const FAULT_INJECTION_DEFINE: &str = "LLAMA_RS_FAULT_INJECTION"; diff --git a/llama-cpp-wrapper-sources/src/lib.rs b/llama-cpp-wrapper-sources/src/lib.rs index 92533a21f..d7aa2ce27 100644 --- a/llama-cpp-wrapper-sources/src/lib.rs +++ b/llama-cpp-wrapper-sources/src/lib.rs @@ -11,6 +11,9 @@ pub mod compile_command; pub mod compile_commands_file; pub mod cpp_standard; +pub mod fault_injection_define; +pub mod system_include_flags; +pub mod vendored_include_dirs; pub mod wrapper_headers; pub mod wrapper_include_dirs; pub mod wrapper_source_paths; diff --git a/llama-cpp-wrapper-sources/src/system_include_flags.rs b/llama-cpp-wrapper-sources/src/system_include_flags.rs new file mode 100644 index 000000000..1a227a65b --- /dev/null +++ b/llama-cpp-wrapper-sources/src/system_include_flags.rs @@ -0,0 +1,35 @@ +use std::path::Path; + +use crate::vendored_include_dirs::VENDORED_INCLUDE_DIRS; + +/// The compiler flags that include every vendored header directory under `sys_dir` as a system +/// directory, so warnings in vendored headers do not surface while ours still do. +#[must_use] +pub fn system_include_flags(sys_dir: &Path) -> Vec { + VENDORED_INCLUDE_DIRS + .iter() + .map(|include_dir| format!("-isystem{}", sys_dir.join(include_dir).display())) + .collect() +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::system_include_flags; + + #[test] + fn every_vendored_include_dir_becomes_a_system_include_under_the_sys_dir() { + assert_eq!( + system_include_flags(Path::new("/repo/llama-cpp-bindings-sys")), + [ + "-isystem/repo/llama-cpp-bindings-sys/GSL/include", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp/common", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp/include", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp/ggml/include", + "-isystem/repo/llama-cpp-bindings-sys/llama.cpp/vendor", + ] + ); + } +} diff --git a/llama-cpp-wrapper-sources/src/vendored_include_dirs.rs b/llama-cpp-wrapper-sources/src/vendored_include_dirs.rs new file mode 100644 index 000000000..ce8eb6c54 --- /dev/null +++ b/llama-cpp-wrapper-sources/src/vendored_include_dirs.rs @@ -0,0 +1,8 @@ +pub const VENDORED_INCLUDE_DIRS: &[&str] = &[ + "GSL/include", + "llama.cpp", + "llama.cpp/common", + "llama.cpp/include", + "llama.cpp/ggml/include", + "llama.cpp/vendor", +]; diff --git a/llama-cpp-wrapper-sources/src/wrapper_headers.rs b/llama-cpp-wrapper-sources/src/wrapper_headers.rs index a366ebd1f..69010918a 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_headers.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_headers.rs @@ -1,17 +1,19 @@ pub const WRAPPER_HEADERS: &[&str] = &[ "wrapper.h", - "wrapper_chat_apply.h", - "wrapper_chat_parse.h", + "wrapper_batch.h", + "wrapper_chat_session.h", + "wrapper_chat_templates.h", + "wrapper_chat_templates_handle.h", "wrapper_common.h", "wrapper_context.h", + "wrapper_fault_injection.h", "wrapper_fit.h", "wrapper_gbnf.h", "wrapper_mtmd.h", - "wrapper_reasoning.h", + "wrapper_parsed_chat.h", + "wrapper_parsed_chat_handle.h", "wrapper_report_current_exception.h", "wrapper_sampler_init.h", "wrapper_state.h", - "wrapper_token_text.h", - "wrapper_tool_calls.h", "wrapper_utils.h", ]; diff --git a/llama-cpp-wrapper-sources/src/wrapper_include_dirs.rs b/llama-cpp-wrapper-sources/src/wrapper_include_dirs.rs index 732fb83a8..316b56433 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_include_dirs.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_include_dirs.rs @@ -1,9 +1 @@ -pub const WRAPPER_INCLUDE_DIRS: &[&str] = &[ - ".", - "GSL/include", - "llama.cpp", - "llama.cpp/common", - "llama.cpp/include", - "llama.cpp/ggml/include", - "llama.cpp/vendor", -]; +pub const WRAPPER_INCLUDE_DIRS: &[&str] = &["."]; diff --git a/llama-cpp-wrapper-sources/src/wrapper_sources.rs b/llama-cpp-wrapper-sources/src/wrapper_sources.rs index 9d231ce60..86d5cb318 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_sources.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_sources.rs @@ -1,14 +1,14 @@ pub const WRAPPER_SOURCES: &[&str] = &[ - "wrapper_chat_apply.cpp", - "wrapper_chat_parse.cpp", + "wrapper_batch.cpp", + "wrapper_chat_session.cpp", + "wrapper_chat_templates.cpp", "wrapper_common.cpp", "wrapper_context.cpp", + "wrapper_fault_injection.cpp", "wrapper_fit.cpp", "wrapper_gbnf.cpp", "wrapper_mtmd.cpp", - "wrapper_reasoning.cpp", + "wrapper_parsed_chat.cpp", "wrapper_sampler_init.cpp", "wrapper_state.cpp", - "wrapper_token_text.cpp", - "wrapper_tool_calls.cpp", ]; diff --git a/llama-cpp-wrapper-sources/src/wrapper_sources_response_file.rs b/llama-cpp-wrapper-sources/src/wrapper_sources_response_file.rs index e3c5cc419..5a34274ed 100644 --- a/llama-cpp-wrapper-sources/src/wrapper_sources_response_file.rs +++ b/llama-cpp-wrapper-sources/src/wrapper_sources_response_file.rs @@ -31,23 +31,24 @@ impl WrapperSourcesResponseFile { #[cfg(test)] mod tests { - use std::path::Path; use std::path::PathBuf; use super::WrapperSourcesResponseFile; - use crate::wrapper_sources_error::WrapperSourcesError; #[test] fn writing_under_a_missing_directory_reports_the_path() { - let output_path = "/nonexistent-directory/wrapper_sources.rsp"; let response_file = WrapperSourcesResponseFile { - output_path: PathBuf::from(output_path), + output_path: PathBuf::from("/nonexistent-directory/wrapper_sources.rsp"), sys_dir: PathBuf::from("/repo/llama-cpp-bindings-sys"), }; - assert!(matches!( - response_file.write(), - Err(WrapperSourcesError::Write { ref path, .. }) if path == Path::new(output_path) - )); + assert_eq!( + response_file.write().err().map(|error| error.to_string()), + Some( + "wrapper source list could not be written to \ + /nonexistent-directory/wrapper_sources.rsp: No such file or directory (os error 2)" + .to_owned() + ) + ); } } diff --git a/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs b/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs index 726f7c085..74b0bfe2a 100644 --- a/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs +++ b/llama-cpp-wrapper-sources/tests/emit_compile_commands.rs @@ -1,6 +1,7 @@ use std::env; use std::fs; use std::path::PathBuf; +use std::process; use std::process::Command; use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; @@ -8,17 +9,18 @@ use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; const EMITTER: &str = env!("CARGO_BIN_EXE_llama-cpp-wrapper-sources"); const SYS_DIR: &str = "/repo/llama-cpp-bindings-sys"; -fn emitter_succeeded(arguments: &[&str]) -> bool { +fn emitter_exit_code(arguments: &[&str]) -> Option { Command::new(EMITTER) .args(arguments) .status() - .is_ok_and(|status| status.success()) + .expect("the emitter binary must run") + .code() } fn temporary_path(suffix: &str) -> PathBuf { env::temp_dir().join(format!( "llama-cpp-wrapper-sources-{}-{suffix}", - std::process::id() + process::id() )) } @@ -27,41 +29,51 @@ fn the_binary_emits_one_entry_and_one_response_line_per_wrapper_source() { let database_path = temporary_path("compile_commands.json"); let response_file_path = temporary_path("wrapper_sources.rsp"); - assert!(emitter_succeeded(&[ - SYS_DIR, - &database_path.display().to_string(), - &response_file_path.display().to_string(), - ])); + assert_eq!( + emitter_exit_code(&[ + SYS_DIR, + &database_path.display().to_string(), + &response_file_path.display().to_string(), + ]), + Some(0) + ); - let database = fs::read_to_string(&database_path).unwrap_or_default(); - let entries: Vec = serde_json::from_str(&database).unwrap_or_default(); + let database = fs::read_to_string(&database_path).expect("the database must be written"); + let entries: Vec = + serde_json::from_str(&database).expect("the database must be a JSON array"); assert_eq!(entries.len(), WRAPPER_SOURCES.len()); - let response_file = fs::read_to_string(&response_file_path).unwrap_or_default(); + let response_file = + fs::read_to_string(&response_file_path).expect("the response file must be written"); assert_eq!(response_file.lines().count(), WRAPPER_SOURCES.len()); - assert!(fs::remove_file(&database_path).is_ok()); - assert!(fs::remove_file(&response_file_path).is_ok()); + fs::remove_file(&database_path).expect("the database must be removable"); + fs::remove_file(&response_file_path).expect("the response file must be removable"); } #[test] fn the_binary_fails_when_the_database_cannot_be_written() { - assert!(!emitter_succeeded(&[ - SYS_DIR, - "/nonexistent-directory/compile_commands.json", - "/nonexistent-directory/wrapper_sources.rsp", - ])); + assert_eq!( + emitter_exit_code(&[ + SYS_DIR, + "/nonexistent-directory/compile_commands.json", + "/nonexistent-directory/wrapper_sources.rsp", + ]), + Some(1) + ); } #[test] -fn the_binary_requires_every_output_path() { +fn the_binary_requires_every_argument() { let database_path = temporary_path("unwritten.json"); - assert!(!emitter_succeeded(&[])); - assert!(!emitter_succeeded(&[SYS_DIR])); - assert!(!emitter_succeeded(&[ - SYS_DIR, - &database_path.display().to_string() - ])); + assert_eq!( + [ + emitter_exit_code(&[]), + emitter_exit_code(&[SYS_DIR]), + emitter_exit_code(&[SYS_DIR, &database_path.display().to_string()]), + ], + [Some(1), Some(1), Some(1)] + ); } diff --git a/llama-cpp-wrapper-sources/tests/wrapper_files_on_disk.rs b/llama-cpp-wrapper-sources/tests/wrapper_files_on_disk.rs new file mode 100644 index 000000000..3890ce356 --- /dev/null +++ b/llama-cpp-wrapper-sources/tests/wrapper_files_on_disk.rs @@ -0,0 +1,39 @@ +use std::collections::BTreeSet; +use std::ffi::OsStr; +use std::fs; +use std::path::Path; + +use llama_cpp_wrapper_sources::wrapper_headers::WRAPPER_HEADERS; +use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES; + +const SYS_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../llama-cpp-bindings-sys"); + +fn sys_root_files_with_extension(extension: &str) -> BTreeSet { + fs::read_dir(Path::new(SYS_DIR)) + .expect("the sys crate directory must be readable") + .map(|entry| { + entry + .expect("a sys crate directory entry must be readable") + .path() + }) + .filter(|path| path.is_file() && path.extension() == Some(OsStr::new(extension))) + .filter_map(|path| path.file_name()?.to_str().map(str::to_owned)) + .collect() +} + +fn listed(files: &[&str]) -> BTreeSet { + files.iter().map(|file| (*file).to_owned()).collect() +} + +#[test] +fn every_cpp_source_in_the_sys_crate_root_is_a_listed_wrapper_source() { + assert_eq!( + sys_root_files_with_extension("cpp"), + listed(WRAPPER_SOURCES) + ); +} + +#[test] +fn every_header_in_the_sys_crate_root_is_a_listed_wrapper_header() { + assert_eq!(sys_root_files_with_extension("h"), listed(WRAPPER_HEADERS)); +}